vortex-nwp 2.0.0b1__py3-none-any.whl → 2.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (141) hide show
  1. vortex/__init__.py +75 -47
  2. vortex/algo/__init__.py +3 -2
  3. vortex/algo/components.py +944 -618
  4. vortex/algo/mpitools.py +802 -497
  5. vortex/algo/mpitools_templates/__init__.py +1 -0
  6. vortex/algo/serversynctools.py +34 -33
  7. vortex/config.py +19 -22
  8. vortex/data/__init__.py +9 -3
  9. vortex/data/abstractstores.py +593 -655
  10. vortex/data/containers.py +217 -162
  11. vortex/data/contents.py +65 -39
  12. vortex/data/executables.py +93 -102
  13. vortex/data/flow.py +40 -34
  14. vortex/data/geometries.py +228 -132
  15. vortex/data/handlers.py +436 -227
  16. vortex/data/outflow.py +15 -15
  17. vortex/data/providers.py +185 -163
  18. vortex/data/resources.py +48 -42
  19. vortex/data/stores.py +540 -417
  20. vortex/data/sync_templates/__init__.py +0 -0
  21. vortex/gloves.py +114 -87
  22. vortex/layout/__init__.py +1 -8
  23. vortex/layout/contexts.py +150 -84
  24. vortex/layout/dataflow.py +353 -202
  25. vortex/layout/monitor.py +264 -128
  26. vortex/nwp/__init__.py +5 -2
  27. vortex/nwp/algo/__init__.py +14 -5
  28. vortex/nwp/algo/assim.py +205 -151
  29. vortex/nwp/algo/clim.py +683 -517
  30. vortex/nwp/algo/coupling.py +447 -225
  31. vortex/nwp/algo/eda.py +437 -229
  32. vortex/nwp/algo/eps.py +403 -231
  33. vortex/nwp/algo/forecasts.py +416 -275
  34. vortex/nwp/algo/fpserver.py +683 -307
  35. vortex/nwp/algo/ifsnaming.py +205 -145
  36. vortex/nwp/algo/ifsroot.py +215 -122
  37. vortex/nwp/algo/monitoring.py +137 -76
  38. vortex/nwp/algo/mpitools.py +330 -190
  39. vortex/nwp/algo/odbtools.py +637 -353
  40. vortex/nwp/algo/oopsroot.py +454 -273
  41. vortex/nwp/algo/oopstests.py +90 -56
  42. vortex/nwp/algo/request.py +287 -206
  43. vortex/nwp/algo/stdpost.py +878 -522
  44. vortex/nwp/data/__init__.py +22 -4
  45. vortex/nwp/data/assim.py +125 -137
  46. vortex/nwp/data/boundaries.py +121 -68
  47. vortex/nwp/data/climfiles.py +193 -211
  48. vortex/nwp/data/configfiles.py +73 -69
  49. vortex/nwp/data/consts.py +426 -401
  50. vortex/nwp/data/ctpini.py +59 -43
  51. vortex/nwp/data/diagnostics.py +94 -66
  52. vortex/nwp/data/eda.py +50 -51
  53. vortex/nwp/data/eps.py +195 -146
  54. vortex/nwp/data/executables.py +440 -434
  55. vortex/nwp/data/fields.py +63 -48
  56. vortex/nwp/data/gridfiles.py +183 -111
  57. vortex/nwp/data/logs.py +250 -217
  58. vortex/nwp/data/modelstates.py +180 -151
  59. vortex/nwp/data/monitoring.py +72 -99
  60. vortex/nwp/data/namelists.py +254 -202
  61. vortex/nwp/data/obs.py +400 -308
  62. vortex/nwp/data/oopsexec.py +22 -20
  63. vortex/nwp/data/providers.py +90 -65
  64. vortex/nwp/data/query.py +71 -82
  65. vortex/nwp/data/stores.py +49 -36
  66. vortex/nwp/data/surfex.py +136 -137
  67. vortex/nwp/syntax/__init__.py +1 -1
  68. vortex/nwp/syntax/stdattrs.py +173 -111
  69. vortex/nwp/tools/__init__.py +2 -2
  70. vortex/nwp/tools/addons.py +22 -17
  71. vortex/nwp/tools/agt.py +24 -12
  72. vortex/nwp/tools/bdap.py +16 -5
  73. vortex/nwp/tools/bdcp.py +4 -1
  74. vortex/nwp/tools/bdm.py +3 -0
  75. vortex/nwp/tools/bdmp.py +14 -9
  76. vortex/nwp/tools/conftools.py +728 -378
  77. vortex/nwp/tools/drhook.py +12 -8
  78. vortex/nwp/tools/grib.py +65 -39
  79. vortex/nwp/tools/gribdiff.py +22 -17
  80. vortex/nwp/tools/ifstools.py +82 -42
  81. vortex/nwp/tools/igastuff.py +167 -143
  82. vortex/nwp/tools/mars.py +14 -2
  83. vortex/nwp/tools/odb.py +234 -125
  84. vortex/nwp/tools/partitioning.py +61 -37
  85. vortex/nwp/tools/satrad.py +27 -12
  86. vortex/nwp/util/async.py +83 -55
  87. vortex/nwp/util/beacon.py +10 -10
  88. vortex/nwp/util/diffpygram.py +174 -86
  89. vortex/nwp/util/ens.py +144 -63
  90. vortex/nwp/util/hooks.py +30 -19
  91. vortex/nwp/util/taskdeco.py +28 -24
  92. vortex/nwp/util/usepygram.py +278 -172
  93. vortex/nwp/util/usetnt.py +31 -17
  94. vortex/sessions.py +72 -39
  95. vortex/syntax/__init__.py +1 -1
  96. vortex/syntax/stdattrs.py +410 -171
  97. vortex/syntax/stddeco.py +31 -22
  98. vortex/toolbox.py +327 -192
  99. vortex/tools/__init__.py +11 -2
  100. vortex/tools/actions.py +110 -121
  101. vortex/tools/addons.py +111 -92
  102. vortex/tools/arm.py +42 -22
  103. vortex/tools/compression.py +72 -69
  104. vortex/tools/date.py +11 -4
  105. vortex/tools/delayedactions.py +242 -132
  106. vortex/tools/env.py +75 -47
  107. vortex/tools/folder.py +342 -171
  108. vortex/tools/grib.py +341 -162
  109. vortex/tools/lfi.py +423 -216
  110. vortex/tools/listings.py +109 -40
  111. vortex/tools/names.py +218 -156
  112. vortex/tools/net.py +655 -299
  113. vortex/tools/parallelism.py +93 -61
  114. vortex/tools/prestaging.py +55 -31
  115. vortex/tools/schedulers.py +172 -105
  116. vortex/tools/services.py +403 -334
  117. vortex/tools/storage.py +293 -358
  118. vortex/tools/surfex.py +24 -24
  119. vortex/tools/systems.py +1234 -643
  120. vortex/tools/targets.py +156 -100
  121. vortex/util/__init__.py +1 -1
  122. vortex/util/config.py +378 -327
  123. vortex/util/empty.py +2 -2
  124. vortex/util/helpers.py +56 -24
  125. vortex/util/introspection.py +18 -12
  126. vortex/util/iosponge.py +8 -4
  127. vortex/util/roles.py +4 -6
  128. vortex/util/storefunctions.py +39 -13
  129. vortex/util/structs.py +3 -3
  130. vortex/util/worker.py +29 -17
  131. vortex_nwp-2.1.0.dist-info/METADATA +67 -0
  132. vortex_nwp-2.1.0.dist-info/RECORD +144 -0
  133. {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.1.0.dist-info}/WHEEL +1 -1
  134. vortex/layout/appconf.py +0 -109
  135. vortex/layout/jobs.py +0 -1276
  136. vortex/layout/nodes.py +0 -1424
  137. vortex/layout/subjobs.py +0 -464
  138. vortex_nwp-2.0.0b1.dist-info/METADATA +0 -50
  139. vortex_nwp-2.0.0b1.dist-info/RECORD +0 -146
  140. {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.1.0.dist-info/licenses}/LICENSE +0 -0
  141. {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.1.0.dist-info}/top_level.txt +0 -0
vortex/nwp/util/usetnt.py CHANGED
@@ -14,7 +14,7 @@ from vortex.util.roles import setrole
14
14
 
15
15
  logger = loggers.getLogger(__name__)
16
16
 
17
- tnt_checker = ExternalCodeImportChecker('thenamelisttool')
17
+ tnt_checker = ExternalCodeImportChecker("thenamelisttool")
18
18
  with tnt_checker as tnt_register:
19
19
  import thenamelisttool
20
20
 
@@ -48,40 +48,54 @@ def compose_nam(options):
48
48
 
49
49
  :rtype: A file like object
50
50
  """
51
- rhdict = options.get('rhandler', None)
52
- source = rhdict['resource'].get('source', None)
51
+ rhdict = options.get("rhandler", None)
52
+ source = rhdict["resource"].get("source", None)
53
53
  if source is None:
54
54
  logger.error("Inapropriate type of resource. Got:\n%s", rhdict)
55
- raise FunctionStoreCallbackError('Inapropriate type of resources.')
55
+ raise FunctionStoreCallbackError("Inapropriate type of resources.")
56
56
 
57
57
  t = sessions.current()
58
58
 
59
59
  def _get_pack_adress(role):
60
60
  role = setrole(role)
61
- packlist = [sec.rh for sec in t.context.sequence.filtered_inputs(role=role)]
61
+ packlist = [
62
+ sec.rh for sec in t.context.sequence.filtered_inputs(role=role)
63
+ ]
62
64
  if len(packlist) != 1:
63
65
  logger.error("The number of namelist packs with role=%s is not 1.")
64
- raise FunctionStoreCallbackError('Incorrect number of namelist packs.')
66
+ raise FunctionStoreCallbackError(
67
+ "Incorrect number of namelist packs."
68
+ )
65
69
  packrh = packlist[0]
66
- if packrh.resource.realkind != 'namelistpack':
67
- logger.error("Incorrect resource type for role %s. Resource handler:\n%s",
68
- role, packrh.icdard())
69
- raise FunctionStoreCallbackError('Incorrect resource type.')
70
+ if packrh.resource.realkind != "namelistpack":
71
+ logger.error(
72
+ "Incorrect resource type for role %s. Resource handler:\n%s",
73
+ role,
74
+ packrh.icdard(),
75
+ )
76
+ raise FunctionStoreCallbackError("Incorrect resource type.")
70
77
  if not packrh.container.filled:
71
- logger.error("The resource handler's container is not filled for role %s", role)
72
- raise FunctionStoreCallbackError('RH container is not filled.')
78
+ logger.error(
79
+ "The resource handler's container is not filled for role %s",
80
+ role,
81
+ )
82
+ raise FunctionStoreCallbackError("RH container is not filled.")
73
83
  return packrh.container.abspath
74
84
 
75
- nampack_path = _get_pack_adress(options.get('nampack', 'Main Namelist Pack'))
76
- dirpack_role = options.get('dirpack', None)
77
- dirpack_path = _get_pack_adress(dirpack_role) if dirpack_role else nampack_path
85
+ nampack_path = _get_pack_adress(
86
+ options.get("nampack", "Main Namelist Pack")
87
+ )
88
+ dirpack_role = options.get("dirpack", None)
89
+ dirpack_path = (
90
+ _get_pack_adress(dirpack_role) if dirpack_role else nampack_path
91
+ )
78
92
 
79
93
  out_io = io.BytesIO()
80
94
  thenamelisttool.util.compose_namelist(
81
- t.sh.path.join(dirpack_path, source + '.yaml'),
95
+ t.sh.path.join(dirpack_path, source + ".yaml"),
82
96
  sourcenam_directory=nampack_path,
83
97
  sorting=thenamelisttool.namadapter.FIRST_ORDER_SORTING,
84
98
  squeeze=False,
85
- fhoutput=codecs.getwriter('utf-8')(out_io)
99
+ fhoutput=codecs.getwriter("utf-8")(out_io),
86
100
  )
87
101
  return out_io
vortex/sessions.py CHANGED
@@ -17,7 +17,7 @@ import footprints
17
17
 
18
18
  from vortex.tools.env import Environment
19
19
 
20
- from vortex import gloves # @UnusedImport
20
+ from vortex import gloves as gloves # footprints import
21
21
  from vortex.layout import contexts
22
22
 
23
23
  #: No automatic export
@@ -28,9 +28,13 @@ logger = loggers.getLogger(__name__)
28
28
 
29
29
  # Module Interface
30
30
 
31
+
31
32
  def get(**kw):
32
33
  """Return actual session ticket object matching description."""
33
- if kw.get('tag', 'current') == 'current' and Ticket.tag_focus() is not None:
34
+ if (
35
+ kw.get("tag", "current") == "current"
36
+ and Ticket.tag_focus() is not None
37
+ ):
34
38
  return current()
35
39
  else:
36
40
  return Ticket(**kw)
@@ -73,7 +77,7 @@ def getglove(**kw):
73
77
 
74
78
  def system(**kw):
75
79
  """Returns the system associated to the current ticket."""
76
- return get(tag=kw.pop('tag', Ticket.tag_focus())).system(**kw)
80
+ return get(tag=kw.pop("tag", Ticket.tag_focus())).system(**kw)
77
81
 
78
82
 
79
83
  # noinspection PyShadowingBuiltins
@@ -94,17 +98,19 @@ class Ticket(getbytag.GetByTag):
94
98
  Default session ticket class, defined by tag.
95
99
  """
96
100
 
97
- _tag_default = 'root'
98
-
99
- def __init__(self,
100
- active=False,
101
- topenv=None,
102
- glove=None,
103
- context=None,
104
- datastore=None,
105
- prompt='Vortex:'):
101
+ _tag_default = "root"
102
+
103
+ def __init__(
104
+ self,
105
+ active=False,
106
+ topenv=None,
107
+ glove=None,
108
+ context=None,
109
+ datastore=None,
110
+ prompt="Vortex:",
111
+ ):
106
112
  self.prompt = prompt
107
- self.line = "\n" + '-' * 100 + "\n"
113
+ self.line = "\n" + "-" * 100 + "\n"
108
114
 
109
115
  self._started = date.now()
110
116
  self._closed = 0
@@ -120,18 +126,24 @@ class Ticket(getbytag.GetByTag):
120
126
  else:
121
127
  self._glove = getglove()
122
128
 
123
- logger.debug('New session system is %s', self.system())
129
+ logger.debug("New session system is %s", self.system())
124
130
 
125
131
  self._rundir = self.sh.getcwd()
126
132
 
127
- logger.debug('Open session %s %s', self.tag, self)
133
+ logger.debug("Open session %s %s", self.tag, self)
128
134
 
129
135
  if datastore is None:
130
- datastore = DataStore(default_picklefile='{:s}_session_datastore.pickled'.format(self.tag))
136
+ datastore = DataStore(
137
+ default_picklefile="{:s}_session_datastore.pickled".format(
138
+ self.tag
139
+ )
140
+ )
131
141
  self._dstore = datastore
132
142
 
133
143
  if context is None:
134
- context = contexts.Context(tag=self.tag, topenv=self._topenv, path=self.path)
144
+ context = contexts.Context(
145
+ tag=self.tag, topenv=self._topenv, path=self.path
146
+ )
135
147
  self._last_context = context
136
148
 
137
149
  if active:
@@ -144,12 +156,20 @@ class Ticket(getbytag.GetByTag):
144
156
  def _set_rundir(self, path):
145
157
  """Set a new default rundir for this session."""
146
158
  if self._rundir:
147
- logger.warning('Session <%s> is changing its working directory <%s>', self.tag, self._rundir)
159
+ logger.warning(
160
+ "Session <%s> is changing its working directory <%s>",
161
+ self.tag,
162
+ self._rundir,
163
+ )
148
164
  if self.sh.path.isdir(path):
149
165
  self._rundir = path
150
- logger.info('Session <%s> set rundir <%s>', self.tag, self._rundir)
166
+ logger.info("Session <%s> set rundir <%s>", self.tag, self._rundir)
151
167
  else:
152
- logger.error('Try to change session <%s> to invalid path <%s>', self.tag, path)
168
+ logger.error(
169
+ "Try to change session <%s> to invalid path <%s>",
170
+ self.tag,
171
+ path,
172
+ )
153
173
 
154
174
  rundir = property(_get_rundir, _set_rundir)
155
175
 
@@ -210,11 +230,14 @@ class Ticket(getbytag.GetByTag):
210
230
  Returns the current OS handler used or set a new one according
211
231
  to ``kw`` dictionary-like arguments.
212
232
  """
213
- refill = kw.pop('refill', False)
233
+ refill = kw.pop("refill", False)
214
234
  if not self._system or kw or refill:
215
235
  self._system = footprints.proxy.system(glove=self.glove, **kw)
216
236
  if not self._system:
217
- logger.critical('Could not load a system object with description %s', str(kw))
237
+ logger.critical(
238
+ "Could not load a system object with description %s",
239
+ str(kw),
240
+ )
218
241
  return self._system
219
242
 
220
243
  def duration(self):
@@ -237,27 +260,31 @@ class Ticket(getbytag.GetByTag):
237
260
  def close(self):
238
261
  """Closes the current session."""
239
262
  if self.closed:
240
- logger.warning('Session %s already closed at %s', self.tag, self.closed)
263
+ logger.warning(
264
+ "Session %s already closed at %s", self.tag, self.closed
265
+ )
241
266
  else:
242
267
  self._closed = date.now()
243
- logger.debug('Close session %s ( time = %s )', self.tag, self.duration())
268
+ logger.debug(
269
+ "Close session %s ( time = %s )", self.tag, self.duration()
270
+ )
244
271
 
245
272
  @property
246
273
  def path(self):
247
- return '/' + self.tag
274
+ return "/" + self.tag
248
275
 
249
276
  @property
250
277
  def subcontexts(self):
251
278
  """The current contexts binded to this session."""
252
- rootpath = self.path + '/'
279
+ rootpath = self.path + "/"
253
280
  return [x for x in contexts.values() if x.path.startswith(rootpath)]
254
281
 
255
282
  def exit(self):
256
283
  """Exit from the current session."""
257
284
  ok = True
258
- logger.debug('Exit session %s %s', self.tag, self)
285
+ logger.debug("Exit session %s %s", self.tag, self)
259
286
  for kid in self.subcontexts:
260
- logger.debug('Exit from context %s', kid)
287
+ logger.debug("Exit from context %s", kid)
261
288
  ok = ok and kid.exit()
262
289
  if self.opened:
263
290
  self.close()
@@ -295,20 +322,26 @@ class Ticket(getbytag.GetByTag):
295
322
  """
296
323
  Returns the logging level.
297
324
  """
298
- v_logger = loggers.getLogger('vortex')
325
+ v_logger = loggers.getLogger("vortex")
299
326
  return logging.getLevelName(v_logger.getEffectiveLevel())
300
327
 
301
- def idcard(self, indent='+ '):
328
+ def idcard(self, indent="+ "):
302
329
  """Returns a printable description of the current session."""
303
- card = "\n".join((
304
- '{0}Name = {1:s}',
305
- '{0}Started = {2!s}',
306
- '{0}Opened = {3!s}',
307
- '{0}Duration = {4!s}',
308
- '{0}Loglevel = {5:s}'
309
- )).format(
330
+ card = "\n".join(
331
+ (
332
+ "{0}Name = {1:s}",
333
+ "{0}Started = {2!s}",
334
+ "{0}Opened = {3!s}",
335
+ "{0}Duration = {4!s}",
336
+ "{0}Loglevel = {5:s}",
337
+ )
338
+ ).format(
310
339
  indent,
311
- self.tag, self.started, self.opened, self.duration(), self.loglevel
340
+ self.tag,
341
+ self.started,
342
+ self.opened,
343
+ self.duration(),
344
+ self.loglevel,
312
345
  )
313
346
  return card
314
347
 
@@ -333,7 +366,7 @@ class Ticket(getbytag.GetByTag):
333
366
  obj.catch_focus()
334
367
  return obj
335
368
  else:
336
- logger.error('Try to switch to an undefined session: %s', tag)
369
+ logger.error("Try to switch to an undefined session: %s", tag)
337
370
  return None
338
371
 
339
372
  def __del__(self):
vortex/syntax/__init__.py CHANGED
@@ -6,4 +6,4 @@ The most important usage is done by :class:`FootprintBase` derivated objects.
6
6
  #: No automatic export
7
7
  __all__ = []
8
8
 
9
- __tocinfoline__ = 'VORTEX package where standard attributes are defined.'
9
+ __tocinfoline__ = "VORTEX package where standard attributes are defined."