vortex-nwp 2.0.0b1__py3-none-any.whl → 2.0.0b2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (139) hide show
  1. vortex/__init__.py +59 -45
  2. vortex/algo/__init__.py +3 -2
  3. vortex/algo/components.py +940 -614
  4. vortex/algo/mpitools.py +802 -497
  5. vortex/algo/serversynctools.py +34 -33
  6. vortex/config.py +19 -22
  7. vortex/data/__init__.py +9 -3
  8. vortex/data/abstractstores.py +593 -655
  9. vortex/data/containers.py +217 -162
  10. vortex/data/contents.py +65 -39
  11. vortex/data/executables.py +93 -102
  12. vortex/data/flow.py +40 -34
  13. vortex/data/geometries.py +228 -132
  14. vortex/data/handlers.py +428 -225
  15. vortex/data/outflow.py +15 -15
  16. vortex/data/providers.py +185 -163
  17. vortex/data/resources.py +48 -42
  18. vortex/data/stores.py +544 -413
  19. vortex/gloves.py +114 -87
  20. vortex/layout/__init__.py +1 -8
  21. vortex/layout/contexts.py +150 -84
  22. vortex/layout/dataflow.py +353 -202
  23. vortex/layout/monitor.py +264 -128
  24. vortex/nwp/__init__.py +5 -2
  25. vortex/nwp/algo/__init__.py +14 -5
  26. vortex/nwp/algo/assim.py +205 -151
  27. vortex/nwp/algo/clim.py +683 -517
  28. vortex/nwp/algo/coupling.py +447 -225
  29. vortex/nwp/algo/eda.py +437 -229
  30. vortex/nwp/algo/eps.py +403 -231
  31. vortex/nwp/algo/forecasts.py +420 -271
  32. vortex/nwp/algo/fpserver.py +683 -307
  33. vortex/nwp/algo/ifsnaming.py +205 -145
  34. vortex/nwp/algo/ifsroot.py +210 -122
  35. vortex/nwp/algo/monitoring.py +132 -76
  36. vortex/nwp/algo/mpitools.py +321 -191
  37. vortex/nwp/algo/odbtools.py +617 -353
  38. vortex/nwp/algo/oopsroot.py +449 -273
  39. vortex/nwp/algo/oopstests.py +90 -56
  40. vortex/nwp/algo/request.py +287 -206
  41. vortex/nwp/algo/stdpost.py +878 -522
  42. vortex/nwp/data/__init__.py +22 -4
  43. vortex/nwp/data/assim.py +125 -137
  44. vortex/nwp/data/boundaries.py +121 -68
  45. vortex/nwp/data/climfiles.py +193 -211
  46. vortex/nwp/data/configfiles.py +73 -69
  47. vortex/nwp/data/consts.py +426 -401
  48. vortex/nwp/data/ctpini.py +59 -43
  49. vortex/nwp/data/diagnostics.py +94 -66
  50. vortex/nwp/data/eda.py +50 -51
  51. vortex/nwp/data/eps.py +195 -146
  52. vortex/nwp/data/executables.py +440 -434
  53. vortex/nwp/data/fields.py +63 -48
  54. vortex/nwp/data/gridfiles.py +183 -111
  55. vortex/nwp/data/logs.py +250 -217
  56. vortex/nwp/data/modelstates.py +180 -151
  57. vortex/nwp/data/monitoring.py +72 -99
  58. vortex/nwp/data/namelists.py +254 -202
  59. vortex/nwp/data/obs.py +400 -308
  60. vortex/nwp/data/oopsexec.py +22 -20
  61. vortex/nwp/data/providers.py +90 -65
  62. vortex/nwp/data/query.py +71 -82
  63. vortex/nwp/data/stores.py +49 -36
  64. vortex/nwp/data/surfex.py +136 -137
  65. vortex/nwp/syntax/__init__.py +1 -1
  66. vortex/nwp/syntax/stdattrs.py +173 -111
  67. vortex/nwp/tools/__init__.py +2 -2
  68. vortex/nwp/tools/addons.py +22 -17
  69. vortex/nwp/tools/agt.py +24 -12
  70. vortex/nwp/tools/bdap.py +16 -5
  71. vortex/nwp/tools/bdcp.py +4 -1
  72. vortex/nwp/tools/bdm.py +3 -0
  73. vortex/nwp/tools/bdmp.py +14 -9
  74. vortex/nwp/tools/conftools.py +728 -378
  75. vortex/nwp/tools/drhook.py +12 -8
  76. vortex/nwp/tools/grib.py +65 -39
  77. vortex/nwp/tools/gribdiff.py +22 -17
  78. vortex/nwp/tools/ifstools.py +82 -42
  79. vortex/nwp/tools/igastuff.py +167 -143
  80. vortex/nwp/tools/mars.py +14 -2
  81. vortex/nwp/tools/odb.py +234 -125
  82. vortex/nwp/tools/partitioning.py +61 -37
  83. vortex/nwp/tools/satrad.py +27 -12
  84. vortex/nwp/util/async.py +83 -55
  85. vortex/nwp/util/beacon.py +10 -10
  86. vortex/nwp/util/diffpygram.py +174 -86
  87. vortex/nwp/util/ens.py +144 -63
  88. vortex/nwp/util/hooks.py +30 -19
  89. vortex/nwp/util/taskdeco.py +28 -24
  90. vortex/nwp/util/usepygram.py +278 -172
  91. vortex/nwp/util/usetnt.py +31 -17
  92. vortex/sessions.py +72 -39
  93. vortex/syntax/__init__.py +1 -1
  94. vortex/syntax/stdattrs.py +410 -171
  95. vortex/syntax/stddeco.py +31 -22
  96. vortex/toolbox.py +327 -192
  97. vortex/tools/__init__.py +11 -2
  98. vortex/tools/actions.py +125 -59
  99. vortex/tools/addons.py +111 -92
  100. vortex/tools/arm.py +42 -22
  101. vortex/tools/compression.py +72 -69
  102. vortex/tools/date.py +11 -4
  103. vortex/tools/delayedactions.py +242 -132
  104. vortex/tools/env.py +75 -47
  105. vortex/tools/folder.py +342 -171
  106. vortex/tools/grib.py +311 -149
  107. vortex/tools/lfi.py +423 -216
  108. vortex/tools/listings.py +109 -40
  109. vortex/tools/names.py +218 -156
  110. vortex/tools/net.py +632 -298
  111. vortex/tools/parallelism.py +93 -61
  112. vortex/tools/prestaging.py +55 -31
  113. vortex/tools/schedulers.py +172 -105
  114. vortex/tools/services.py +402 -333
  115. vortex/tools/storage.py +293 -358
  116. vortex/tools/surfex.py +24 -24
  117. vortex/tools/systems.py +1211 -631
  118. vortex/tools/targets.py +156 -100
  119. vortex/util/__init__.py +1 -1
  120. vortex/util/config.py +377 -327
  121. vortex/util/empty.py +2 -2
  122. vortex/util/helpers.py +56 -24
  123. vortex/util/introspection.py +18 -12
  124. vortex/util/iosponge.py +8 -4
  125. vortex/util/roles.py +4 -6
  126. vortex/util/storefunctions.py +39 -13
  127. vortex/util/structs.py +3 -3
  128. vortex/util/worker.py +29 -17
  129. vortex_nwp-2.0.0b2.dist-info/METADATA +66 -0
  130. vortex_nwp-2.0.0b2.dist-info/RECORD +142 -0
  131. {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.0.0b2.dist-info}/WHEEL +1 -1
  132. vortex/layout/appconf.py +0 -109
  133. vortex/layout/jobs.py +0 -1276
  134. vortex/layout/nodes.py +0 -1424
  135. vortex/layout/subjobs.py +0 -464
  136. vortex_nwp-2.0.0b1.dist-info/METADATA +0 -50
  137. vortex_nwp-2.0.0b1.dist-info/RECORD +0 -146
  138. {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.0.0b2.dist-info}/LICENSE +0 -0
  139. {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.0.0b2.dist-info}/top_level.txt +0 -0
vortex/tools/env.py CHANGED
@@ -19,10 +19,12 @@ __all__ = []
19
19
  logger = loggers.getLogger(__name__)
20
20
 
21
21
  #: Pre-compiled evaluation mostly used by :class:`Environment` method (true).
22
- vartrue = re.compile(r'^\s*(?:[1-9]\d*|ok|on|true|yes|y)\s*$', flags=re.IGNORECASE)
22
+ vartrue = re.compile(
23
+ r"^\s*(?:[1-9]\d*|ok|on|true|yes|y)\s*$", flags=re.IGNORECASE
24
+ )
23
25
 
24
26
  #: Pre-compiled evaluation mostly used by :class:`Environment` method (false).
25
- varfalse = re.compile(r'^\s*(?:0|ko|off|false|no|n)\s*$', flags=re.IGNORECASE)
27
+ varfalse = re.compile(r"^\s*(?:0|ko|off|false|no|n)\s*$", flags=re.IGNORECASE)
26
28
 
27
29
 
28
30
  def current():
@@ -51,8 +53,16 @@ class Environment:
51
53
 
52
54
  _current_active = None
53
55
 
54
- def __init__(self, env=None, active=False, clear=False, verbose=False,
55
- noexport=[], contextlock=None, history=True):
56
+ def __init__(
57
+ self,
58
+ env=None,
59
+ active=False,
60
+ clear=False,
61
+ verbose=False,
62
+ noexport=[],
63
+ contextlock=None,
64
+ history=True,
65
+ ):
56
66
  """
57
67
  :param Environment env: An existing Environment used to initialise this object.
58
68
  :param bool active: Is this new environment activated when created.
@@ -69,18 +79,18 @@ class Environment:
69
79
  :class:`~bronx.stdtypes.history.PrivateHistory` object that will be
70
80
  accessible through the :attr:`history` property.
71
81
  """
72
- self.__dict__['_history'] = PrivateHistory() if history else None
73
- self.__dict__['_verbose'] = verbose
74
- self.__dict__['_frozen'] = collections.deque()
75
- self.__dict__['_pool'] = dict()
76
- self.__dict__['_mods'] = set()
77
- self.__dict__['_sh'] = None
78
- self.__dict__['_os'] = list()
82
+ self.__dict__["_history"] = PrivateHistory() if history else None
83
+ self.__dict__["_verbose"] = verbose
84
+ self.__dict__["_frozen"] = collections.deque()
85
+ self.__dict__["_pool"] = dict()
86
+ self.__dict__["_mods"] = set()
87
+ self.__dict__["_sh"] = None
88
+ self.__dict__["_os"] = list()
79
89
  if env is not None and isinstance(env, Environment):
80
90
  self._env_clone_internals(env, contextlock)
81
91
  if verbose:
82
92
  try:
83
- self.__dict__['_sh'] = env._sh
93
+ self.__dict__["_sh"] = env._sh
84
94
  except AttributeError:
85
95
  pass
86
96
  else:
@@ -88,20 +98,22 @@ class Environment:
88
98
  active = False
89
99
  else:
90
100
  if self._current_active is not None:
91
- self._env_clone_internals(self._current_active, contextlock)
101
+ self._env_clone_internals(
102
+ self._current_active, contextlock
103
+ )
92
104
  else:
93
105
  self._pool.update(os.environ)
94
- self.__dict__['_noexport'] = [x.upper() for x in noexport]
106
+ self.__dict__["_noexport"] = [x.upper() for x in noexport]
95
107
  self.active(active)
96
108
 
97
109
  def _env_clone_internals(self, env, contextlock):
98
- self.__dict__['_os'] = env.osstack()
99
- self.__dict__['_os'].append(env)
110
+ self.__dict__["_os"] = env.osstack()
111
+ self.__dict__["_os"].append(env)
100
112
  self._pool.update(env.items())
101
113
  if contextlock is not None:
102
- self.__dict__['_contextlock'] = contextlock
114
+ self.__dict__["_contextlock"] = contextlock
103
115
  else:
104
- self.__dict__['_contextlock'] = env.contextlock
116
+ self.__dict__["_contextlock"] = env.contextlock
105
117
 
106
118
  @property
107
119
  def history(self):
@@ -116,7 +128,9 @@ class Environment:
116
128
  self.history.append(var, value, traceback.format_stack()[:-1])
117
129
 
118
130
  def __str__(self):
119
- return '{:s} | including {:d} variables>'.format(repr(self).rstrip('>'), len(self))
131
+ return "{:s} | including {:d} variables>".format(
132
+ repr(self).rstrip(">"), len(self)
133
+ )
120
134
 
121
135
  @classmethod
122
136
  def current(cls):
@@ -139,11 +153,11 @@ class Environment:
139
153
  """Dump the specified ``value`` as a string (utility function)."""
140
154
  if isinstance(value, str):
141
155
  obj = str(value)
142
- elif hasattr(value, 'export_dict'):
156
+ elif hasattr(value, "export_dict"):
143
157
  obj = value.export_dict()
144
- elif hasattr(value, 'footprint_export'):
158
+ elif hasattr(value, "footprint_export"):
145
159
  obj = value.footprint_export()
146
- elif hasattr(value, '__dict__'):
160
+ elif hasattr(value, "__dict__"):
147
161
  obj = vars(value)
148
162
  else:
149
163
  obj = value
@@ -169,7 +183,9 @@ class Environment:
169
183
  os.environ[upvar] = actualvalue
170
184
  if self.verbose():
171
185
  if self.osbound() and self._sh:
172
- self._sh.stderr('export', '{:s}={:s}'.format(upvar, actualvalue))
186
+ self._sh.stderr(
187
+ "export", "{:s}={:s}".format(upvar, actualvalue)
188
+ )
173
189
  logger.debug('Env export %s="%s"', upvar, actualvalue)
174
190
 
175
191
  def __setitem__(self, varname, value):
@@ -195,7 +211,7 @@ class Environment:
195
211
  return self.getvar(varname)
196
212
 
197
213
  def __getattr__(self, varname):
198
- if varname.startswith('_'):
214
+ if varname.startswith("_"):
199
215
  raise AttributeError
200
216
  else:
201
217
  return self.getvar(varname)
@@ -217,9 +233,9 @@ class Environment:
217
233
  if seen and self.osbound():
218
234
  del os.environ[varname.upper()]
219
235
  if self.verbose() and self._sh:
220
- self._sh.stderr('unset', '{:s}'.format(varname.upper()))
236
+ self._sh.stderr("unset", "{:s}".format(varname.upper()))
221
237
  if seen:
222
- self._record(varname.upper(), '!!deleted!!')
238
+ self._record(varname.upper(), "!!deleted!!")
223
239
 
224
240
  def __delitem__(self, varname):
225
241
  self.delvar(varname)
@@ -268,9 +284,11 @@ class Environment:
268
284
  return self._pool.items()
269
285
 
270
286
  def __eq__(self, other):
271
- return (isinstance(other, type(self)) and
272
- set(self.keys()) == set(other.keys) and
273
- all([self[k] == other[k] for k in self.keys]))
287
+ return (
288
+ isinstance(other, type(self))
289
+ and set(self.keys()) == set(other.keys)
290
+ and all([self[k] == other[k] for k in self.keys])
291
+ )
274
292
 
275
293
  def __ne__(self, other):
276
294
  return not self == other
@@ -342,7 +360,9 @@ class Environment:
342
360
  try:
343
361
  eclone.verbose(self._verbose, self._sh)
344
362
  except AttributeError:
345
- logger.debug('Could not find verbose attributes while cloning env...')
363
+ logger.debug(
364
+ "Could not find verbose attributes while cloning env..."
365
+ )
346
366
  return eclone
347
367
 
348
368
  def __enter__(self):
@@ -365,10 +385,10 @@ class Environment:
365
385
  def verbose(self, switch=None, sh=None, fromenv=None):
366
386
  """Switch on or off the verbose mode. Returns actual value."""
367
387
  if switch is not None:
368
- self.__dict__['_verbose'] = bool(switch)
388
+ self.__dict__["_verbose"] = bool(switch)
369
389
  if sh is not None:
370
- self.__dict__['_sh'] = sh
371
- return self.__dict__['_verbose']
390
+ self.__dict__["_sh"] = sh
391
+ return self.__dict__["_verbose"]
372
392
 
373
393
  def active(self, *args):
374
394
  """
@@ -383,19 +403,23 @@ class Environment:
383
403
  if args and type(args[0]) is bool:
384
404
  active = args[0]
385
405
  if previous_act and not active and self._os:
386
- self._record('!!OS_BINDING!!', 'Broken...')
406
+ self._record("!!OS_BINDING!!", "Broken...")
387
407
  self.__class__._current_active = self._os[-1]
388
408
  osrewind = self.__class__._current_active
389
409
  if not previous_act and active:
390
410
  if self.contextlock is not None and not self.contextlock.active:
391
- raise RuntimeError("It's not allowed to switch to an Environment " +
392
- "that belongs to an inactive context")
393
- self._record('!!OS_BINDING!!', 'Acquiring...')
411
+ raise RuntimeError(
412
+ "It's not allowed to switch to an Environment "
413
+ + "that belongs to an inactive context"
414
+ )
415
+ self._record("!!OS_BINDING!!", "Acquiring...")
394
416
  self.__class__._current_active = self
395
417
  osrewind = self.__class__._current_active
396
418
  if osrewind:
397
419
  os.environ.clear()
398
- for k in filter(lambda x: x not in osrewind._noexport, osrewind._pool.keys()):
420
+ for k in filter(
421
+ lambda x: x not in osrewind._noexport, osrewind._pool.keys()
422
+ ):
399
423
  os.environ[k] = osrewind.native(k)
400
424
  return active
401
425
 
@@ -436,11 +460,15 @@ class Environment:
436
460
 
437
461
  def mkautolist(self, prefix):
438
462
  """Return a list of variable starting with the ``prefix`` string."""
439
- return [var + '="' + self.get(var, '') + '"' for var in self.keys() if var.startswith(prefix)]
463
+ return [
464
+ var + '="' + self.get(var, "") + '"'
465
+ for var in self.keys()
466
+ if var.startswith(prefix)
467
+ ]
440
468
 
441
469
  def trueshell(self):
442
470
  """Extract the actual shell name according to env variable SHELL."""
443
- return re.sub('^.*/', '', self.getvar('shell'))
471
+ return re.sub("^.*/", "", self.getvar("shell"))
444
472
 
445
473
  def true(self, varname):
446
474
  """Extended boolean positive test of the variable given as argument."""
@@ -461,41 +489,41 @@ class Environment:
461
489
  :param str value: The value that will be inserted in the path
462
490
  :param int pos: Where in the path, to insert ``value`` (by default, at the end)
463
491
  """
464
- mypath = self.getvar(var).split(':') if self.getvar(var) else []
492
+ mypath = self.getvar(var).split(":") if self.getvar(var) else []
465
493
  value = str(value)
466
494
  while value in mypath:
467
495
  mypath.remove(value)
468
496
  if pos is None:
469
497
  pos = len(mypath)
470
498
  mypath.insert(pos, value)
471
- self.setvar(var, ':'.join(mypath))
499
+ self.setvar(var, ":".join(mypath))
472
500
 
473
501
  def rmgenericpath(self, var, value):
474
502
  """Remove the specified ``value`` from a PATH like variable."""
475
- mypath = self.getvar(var).split(':') if self.getvar(var) else []
503
+ mypath = self.getvar(var).split(":") if self.getvar(var) else []
476
504
  while value in mypath:
477
505
  mypath.remove(value)
478
- self.setvar(var, ':'.join(mypath))
506
+ self.setvar(var, ":".join(mypath))
479
507
 
480
508
  def setbinpath(self, value, pos=None):
481
509
  """
482
510
  Insert a new path ``value`` to the bin search path (i.e. the PATH
483
511
  environment variable) at given position.
484
512
  """
485
- self.setgenericpath('PATH', value, pos)
513
+ self.setgenericpath("PATH", value, pos)
486
514
 
487
515
  def rmbinpath(self, value):
488
516
  """
489
517
  Remove the specified ``value`` from the bin path (i.e. the PATH
490
518
  environment variable).
491
519
  """
492
- self.rmgenericpath('PATH', value)
520
+ self.rmgenericpath("PATH", value)
493
521
 
494
522
 
495
523
  collections.abc.Mapping.register(Environment)
496
524
 
497
525
 
498
- class EnvironmentDeltaContext():
526
+ class EnvironmentDeltaContext:
499
527
  """Context that will apply a delta on the Environment and rewind it on exit."""
500
528
 
501
529
  def __init__(self, env, **kw):