vortex-nwp 2.0.0b1__py3-none-any.whl → 2.0.0b2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (139) hide show
  1. vortex/__init__.py +59 -45
  2. vortex/algo/__init__.py +3 -2
  3. vortex/algo/components.py +940 -614
  4. vortex/algo/mpitools.py +802 -497
  5. vortex/algo/serversynctools.py +34 -33
  6. vortex/config.py +19 -22
  7. vortex/data/__init__.py +9 -3
  8. vortex/data/abstractstores.py +593 -655
  9. vortex/data/containers.py +217 -162
  10. vortex/data/contents.py +65 -39
  11. vortex/data/executables.py +93 -102
  12. vortex/data/flow.py +40 -34
  13. vortex/data/geometries.py +228 -132
  14. vortex/data/handlers.py +428 -225
  15. vortex/data/outflow.py +15 -15
  16. vortex/data/providers.py +185 -163
  17. vortex/data/resources.py +48 -42
  18. vortex/data/stores.py +544 -413
  19. vortex/gloves.py +114 -87
  20. vortex/layout/__init__.py +1 -8
  21. vortex/layout/contexts.py +150 -84
  22. vortex/layout/dataflow.py +353 -202
  23. vortex/layout/monitor.py +264 -128
  24. vortex/nwp/__init__.py +5 -2
  25. vortex/nwp/algo/__init__.py +14 -5
  26. vortex/nwp/algo/assim.py +205 -151
  27. vortex/nwp/algo/clim.py +683 -517
  28. vortex/nwp/algo/coupling.py +447 -225
  29. vortex/nwp/algo/eda.py +437 -229
  30. vortex/nwp/algo/eps.py +403 -231
  31. vortex/nwp/algo/forecasts.py +420 -271
  32. vortex/nwp/algo/fpserver.py +683 -307
  33. vortex/nwp/algo/ifsnaming.py +205 -145
  34. vortex/nwp/algo/ifsroot.py +210 -122
  35. vortex/nwp/algo/monitoring.py +132 -76
  36. vortex/nwp/algo/mpitools.py +321 -191
  37. vortex/nwp/algo/odbtools.py +617 -353
  38. vortex/nwp/algo/oopsroot.py +449 -273
  39. vortex/nwp/algo/oopstests.py +90 -56
  40. vortex/nwp/algo/request.py +287 -206
  41. vortex/nwp/algo/stdpost.py +878 -522
  42. vortex/nwp/data/__init__.py +22 -4
  43. vortex/nwp/data/assim.py +125 -137
  44. vortex/nwp/data/boundaries.py +121 -68
  45. vortex/nwp/data/climfiles.py +193 -211
  46. vortex/nwp/data/configfiles.py +73 -69
  47. vortex/nwp/data/consts.py +426 -401
  48. vortex/nwp/data/ctpini.py +59 -43
  49. vortex/nwp/data/diagnostics.py +94 -66
  50. vortex/nwp/data/eda.py +50 -51
  51. vortex/nwp/data/eps.py +195 -146
  52. vortex/nwp/data/executables.py +440 -434
  53. vortex/nwp/data/fields.py +63 -48
  54. vortex/nwp/data/gridfiles.py +183 -111
  55. vortex/nwp/data/logs.py +250 -217
  56. vortex/nwp/data/modelstates.py +180 -151
  57. vortex/nwp/data/monitoring.py +72 -99
  58. vortex/nwp/data/namelists.py +254 -202
  59. vortex/nwp/data/obs.py +400 -308
  60. vortex/nwp/data/oopsexec.py +22 -20
  61. vortex/nwp/data/providers.py +90 -65
  62. vortex/nwp/data/query.py +71 -82
  63. vortex/nwp/data/stores.py +49 -36
  64. vortex/nwp/data/surfex.py +136 -137
  65. vortex/nwp/syntax/__init__.py +1 -1
  66. vortex/nwp/syntax/stdattrs.py +173 -111
  67. vortex/nwp/tools/__init__.py +2 -2
  68. vortex/nwp/tools/addons.py +22 -17
  69. vortex/nwp/tools/agt.py +24 -12
  70. vortex/nwp/tools/bdap.py +16 -5
  71. vortex/nwp/tools/bdcp.py +4 -1
  72. vortex/nwp/tools/bdm.py +3 -0
  73. vortex/nwp/tools/bdmp.py +14 -9
  74. vortex/nwp/tools/conftools.py +728 -378
  75. vortex/nwp/tools/drhook.py +12 -8
  76. vortex/nwp/tools/grib.py +65 -39
  77. vortex/nwp/tools/gribdiff.py +22 -17
  78. vortex/nwp/tools/ifstools.py +82 -42
  79. vortex/nwp/tools/igastuff.py +167 -143
  80. vortex/nwp/tools/mars.py +14 -2
  81. vortex/nwp/tools/odb.py +234 -125
  82. vortex/nwp/tools/partitioning.py +61 -37
  83. vortex/nwp/tools/satrad.py +27 -12
  84. vortex/nwp/util/async.py +83 -55
  85. vortex/nwp/util/beacon.py +10 -10
  86. vortex/nwp/util/diffpygram.py +174 -86
  87. vortex/nwp/util/ens.py +144 -63
  88. vortex/nwp/util/hooks.py +30 -19
  89. vortex/nwp/util/taskdeco.py +28 -24
  90. vortex/nwp/util/usepygram.py +278 -172
  91. vortex/nwp/util/usetnt.py +31 -17
  92. vortex/sessions.py +72 -39
  93. vortex/syntax/__init__.py +1 -1
  94. vortex/syntax/stdattrs.py +410 -171
  95. vortex/syntax/stddeco.py +31 -22
  96. vortex/toolbox.py +327 -192
  97. vortex/tools/__init__.py +11 -2
  98. vortex/tools/actions.py +125 -59
  99. vortex/tools/addons.py +111 -92
  100. vortex/tools/arm.py +42 -22
  101. vortex/tools/compression.py +72 -69
  102. vortex/tools/date.py +11 -4
  103. vortex/tools/delayedactions.py +242 -132
  104. vortex/tools/env.py +75 -47
  105. vortex/tools/folder.py +342 -171
  106. vortex/tools/grib.py +311 -149
  107. vortex/tools/lfi.py +423 -216
  108. vortex/tools/listings.py +109 -40
  109. vortex/tools/names.py +218 -156
  110. vortex/tools/net.py +632 -298
  111. vortex/tools/parallelism.py +93 -61
  112. vortex/tools/prestaging.py +55 -31
  113. vortex/tools/schedulers.py +172 -105
  114. vortex/tools/services.py +402 -333
  115. vortex/tools/storage.py +293 -358
  116. vortex/tools/surfex.py +24 -24
  117. vortex/tools/systems.py +1211 -631
  118. vortex/tools/targets.py +156 -100
  119. vortex/util/__init__.py +1 -1
  120. vortex/util/config.py +377 -327
  121. vortex/util/empty.py +2 -2
  122. vortex/util/helpers.py +56 -24
  123. vortex/util/introspection.py +18 -12
  124. vortex/util/iosponge.py +8 -4
  125. vortex/util/roles.py +4 -6
  126. vortex/util/storefunctions.py +39 -13
  127. vortex/util/structs.py +3 -3
  128. vortex/util/worker.py +29 -17
  129. vortex_nwp-2.0.0b2.dist-info/METADATA +66 -0
  130. vortex_nwp-2.0.0b2.dist-info/RECORD +142 -0
  131. {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.0.0b2.dist-info}/WHEEL +1 -1
  132. vortex/layout/appconf.py +0 -109
  133. vortex/layout/jobs.py +0 -1276
  134. vortex/layout/nodes.py +0 -1424
  135. vortex/layout/subjobs.py +0 -464
  136. vortex_nwp-2.0.0b1.dist-info/METADATA +0 -50
  137. vortex_nwp-2.0.0b1.dist-info/RECORD +0 -146
  138. {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.0.0b2.dist-info}/LICENSE +0 -0
  139. {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.0.0b2.dist-info}/top_level.txt +0 -0
vortex/nwp/util/usetnt.py CHANGED
@@ -14,7 +14,7 @@ from vortex.util.roles import setrole
14
14
 
15
15
  logger = loggers.getLogger(__name__)
16
16
 
17
- tnt_checker = ExternalCodeImportChecker('thenamelisttool')
17
+ tnt_checker = ExternalCodeImportChecker("thenamelisttool")
18
18
  with tnt_checker as tnt_register:
19
19
  import thenamelisttool
20
20
 
@@ -48,40 +48,54 @@ def compose_nam(options):
48
48
 
49
49
  :rtype: A file like object
50
50
  """
51
- rhdict = options.get('rhandler', None)
52
- source = rhdict['resource'].get('source', None)
51
+ rhdict = options.get("rhandler", None)
52
+ source = rhdict["resource"].get("source", None)
53
53
  if source is None:
54
54
  logger.error("Inapropriate type of resource. Got:\n%s", rhdict)
55
- raise FunctionStoreCallbackError('Inapropriate type of resources.')
55
+ raise FunctionStoreCallbackError("Inapropriate type of resources.")
56
56
 
57
57
  t = sessions.current()
58
58
 
59
59
  def _get_pack_adress(role):
60
60
  role = setrole(role)
61
- packlist = [sec.rh for sec in t.context.sequence.filtered_inputs(role=role)]
61
+ packlist = [
62
+ sec.rh for sec in t.context.sequence.filtered_inputs(role=role)
63
+ ]
62
64
  if len(packlist) != 1:
63
65
  logger.error("The number of namelist packs with role=%s is not 1.")
64
- raise FunctionStoreCallbackError('Incorrect number of namelist packs.')
66
+ raise FunctionStoreCallbackError(
67
+ "Incorrect number of namelist packs."
68
+ )
65
69
  packrh = packlist[0]
66
- if packrh.resource.realkind != 'namelistpack':
67
- logger.error("Incorrect resource type for role %s. Resource handler:\n%s",
68
- role, packrh.icdard())
69
- raise FunctionStoreCallbackError('Incorrect resource type.')
70
+ if packrh.resource.realkind != "namelistpack":
71
+ logger.error(
72
+ "Incorrect resource type for role %s. Resource handler:\n%s",
73
+ role,
74
+ packrh.icdard(),
75
+ )
76
+ raise FunctionStoreCallbackError("Incorrect resource type.")
70
77
  if not packrh.container.filled:
71
- logger.error("The resource handler's container is not filled for role %s", role)
72
- raise FunctionStoreCallbackError('RH container is not filled.')
78
+ logger.error(
79
+ "The resource handler's container is not filled for role %s",
80
+ role,
81
+ )
82
+ raise FunctionStoreCallbackError("RH container is not filled.")
73
83
  return packrh.container.abspath
74
84
 
75
- nampack_path = _get_pack_adress(options.get('nampack', 'Main Namelist Pack'))
76
- dirpack_role = options.get('dirpack', None)
77
- dirpack_path = _get_pack_adress(dirpack_role) if dirpack_role else nampack_path
85
+ nampack_path = _get_pack_adress(
86
+ options.get("nampack", "Main Namelist Pack")
87
+ )
88
+ dirpack_role = options.get("dirpack", None)
89
+ dirpack_path = (
90
+ _get_pack_adress(dirpack_role) if dirpack_role else nampack_path
91
+ )
78
92
 
79
93
  out_io = io.BytesIO()
80
94
  thenamelisttool.util.compose_namelist(
81
- t.sh.path.join(dirpack_path, source + '.yaml'),
95
+ t.sh.path.join(dirpack_path, source + ".yaml"),
82
96
  sourcenam_directory=nampack_path,
83
97
  sorting=thenamelisttool.namadapter.FIRST_ORDER_SORTING,
84
98
  squeeze=False,
85
- fhoutput=codecs.getwriter('utf-8')(out_io)
99
+ fhoutput=codecs.getwriter("utf-8")(out_io),
86
100
  )
87
101
  return out_io
vortex/sessions.py CHANGED
@@ -17,7 +17,7 @@ import footprints
17
17
 
18
18
  from vortex.tools.env import Environment
19
19
 
20
- from vortex import gloves # @UnusedImport
20
+ from vortex import gloves as gloves # footprints import
21
21
  from vortex.layout import contexts
22
22
 
23
23
  #: No automatic export
@@ -28,9 +28,13 @@ logger = loggers.getLogger(__name__)
28
28
 
29
29
  # Module Interface
30
30
 
31
+
31
32
  def get(**kw):
32
33
  """Return actual session ticket object matching description."""
33
- if kw.get('tag', 'current') == 'current' and Ticket.tag_focus() is not None:
34
+ if (
35
+ kw.get("tag", "current") == "current"
36
+ and Ticket.tag_focus() is not None
37
+ ):
34
38
  return current()
35
39
  else:
36
40
  return Ticket(**kw)
@@ -73,7 +77,7 @@ def getglove(**kw):
73
77
 
74
78
  def system(**kw):
75
79
  """Returns the system associated to the current ticket."""
76
- return get(tag=kw.pop('tag', Ticket.tag_focus())).system(**kw)
80
+ return get(tag=kw.pop("tag", Ticket.tag_focus())).system(**kw)
77
81
 
78
82
 
79
83
  # noinspection PyShadowingBuiltins
@@ -94,17 +98,19 @@ class Ticket(getbytag.GetByTag):
94
98
  Default session ticket class, defined by tag.
95
99
  """
96
100
 
97
- _tag_default = 'root'
98
-
99
- def __init__(self,
100
- active=False,
101
- topenv=None,
102
- glove=None,
103
- context=None,
104
- datastore=None,
105
- prompt='Vortex:'):
101
+ _tag_default = "root"
102
+
103
+ def __init__(
104
+ self,
105
+ active=False,
106
+ topenv=None,
107
+ glove=None,
108
+ context=None,
109
+ datastore=None,
110
+ prompt="Vortex:",
111
+ ):
106
112
  self.prompt = prompt
107
- self.line = "\n" + '-' * 100 + "\n"
113
+ self.line = "\n" + "-" * 100 + "\n"
108
114
 
109
115
  self._started = date.now()
110
116
  self._closed = 0
@@ -120,18 +126,24 @@ class Ticket(getbytag.GetByTag):
120
126
  else:
121
127
  self._glove = getglove()
122
128
 
123
- logger.debug('New session system is %s', self.system())
129
+ logger.debug("New session system is %s", self.system())
124
130
 
125
131
  self._rundir = self.sh.getcwd()
126
132
 
127
- logger.debug('Open session %s %s', self.tag, self)
133
+ logger.debug("Open session %s %s", self.tag, self)
128
134
 
129
135
  if datastore is None:
130
- datastore = DataStore(default_picklefile='{:s}_session_datastore.pickled'.format(self.tag))
136
+ datastore = DataStore(
137
+ default_picklefile="{:s}_session_datastore.pickled".format(
138
+ self.tag
139
+ )
140
+ )
131
141
  self._dstore = datastore
132
142
 
133
143
  if context is None:
134
- context = contexts.Context(tag=self.tag, topenv=self._topenv, path=self.path)
144
+ context = contexts.Context(
145
+ tag=self.tag, topenv=self._topenv, path=self.path
146
+ )
135
147
  self._last_context = context
136
148
 
137
149
  if active:
@@ -144,12 +156,20 @@ class Ticket(getbytag.GetByTag):
144
156
  def _set_rundir(self, path):
145
157
  """Set a new default rundir for this session."""
146
158
  if self._rundir:
147
- logger.warning('Session <%s> is changing its working directory <%s>', self.tag, self._rundir)
159
+ logger.warning(
160
+ "Session <%s> is changing its working directory <%s>",
161
+ self.tag,
162
+ self._rundir,
163
+ )
148
164
  if self.sh.path.isdir(path):
149
165
  self._rundir = path
150
- logger.info('Session <%s> set rundir <%s>', self.tag, self._rundir)
166
+ logger.info("Session <%s> set rundir <%s>", self.tag, self._rundir)
151
167
  else:
152
- logger.error('Try to change session <%s> to invalid path <%s>', self.tag, path)
168
+ logger.error(
169
+ "Try to change session <%s> to invalid path <%s>",
170
+ self.tag,
171
+ path,
172
+ )
153
173
 
154
174
  rundir = property(_get_rundir, _set_rundir)
155
175
 
@@ -210,11 +230,14 @@ class Ticket(getbytag.GetByTag):
210
230
  Returns the current OS handler used or set a new one according
211
231
  to ``kw`` dictionary-like arguments.
212
232
  """
213
- refill = kw.pop('refill', False)
233
+ refill = kw.pop("refill", False)
214
234
  if not self._system or kw or refill:
215
235
  self._system = footprints.proxy.system(glove=self.glove, **kw)
216
236
  if not self._system:
217
- logger.critical('Could not load a system object with description %s', str(kw))
237
+ logger.critical(
238
+ "Could not load a system object with description %s",
239
+ str(kw),
240
+ )
218
241
  return self._system
219
242
 
220
243
  def duration(self):
@@ -237,27 +260,31 @@ class Ticket(getbytag.GetByTag):
237
260
  def close(self):
238
261
  """Closes the current session."""
239
262
  if self.closed:
240
- logger.warning('Session %s already closed at %s', self.tag, self.closed)
263
+ logger.warning(
264
+ "Session %s already closed at %s", self.tag, self.closed
265
+ )
241
266
  else:
242
267
  self._closed = date.now()
243
- logger.debug('Close session %s ( time = %s )', self.tag, self.duration())
268
+ logger.debug(
269
+ "Close session %s ( time = %s )", self.tag, self.duration()
270
+ )
244
271
 
245
272
  @property
246
273
  def path(self):
247
- return '/' + self.tag
274
+ return "/" + self.tag
248
275
 
249
276
  @property
250
277
  def subcontexts(self):
251
278
  """The current contexts binded to this session."""
252
- rootpath = self.path + '/'
279
+ rootpath = self.path + "/"
253
280
  return [x for x in contexts.values() if x.path.startswith(rootpath)]
254
281
 
255
282
  def exit(self):
256
283
  """Exit from the current session."""
257
284
  ok = True
258
- logger.debug('Exit session %s %s', self.tag, self)
285
+ logger.debug("Exit session %s %s", self.tag, self)
259
286
  for kid in self.subcontexts:
260
- logger.debug('Exit from context %s', kid)
287
+ logger.debug("Exit from context %s", kid)
261
288
  ok = ok and kid.exit()
262
289
  if self.opened:
263
290
  self.close()
@@ -295,20 +322,26 @@ class Ticket(getbytag.GetByTag):
295
322
  """
296
323
  Returns the logging level.
297
324
  """
298
- v_logger = loggers.getLogger('vortex')
325
+ v_logger = loggers.getLogger("vortex")
299
326
  return logging.getLevelName(v_logger.getEffectiveLevel())
300
327
 
301
- def idcard(self, indent='+ '):
328
+ def idcard(self, indent="+ "):
302
329
  """Returns a printable description of the current session."""
303
- card = "\n".join((
304
- '{0}Name = {1:s}',
305
- '{0}Started = {2!s}',
306
- '{0}Opened = {3!s}',
307
- '{0}Duration = {4!s}',
308
- '{0}Loglevel = {5:s}'
309
- )).format(
330
+ card = "\n".join(
331
+ (
332
+ "{0}Name = {1:s}",
333
+ "{0}Started = {2!s}",
334
+ "{0}Opened = {3!s}",
335
+ "{0}Duration = {4!s}",
336
+ "{0}Loglevel = {5:s}",
337
+ )
338
+ ).format(
310
339
  indent,
311
- self.tag, self.started, self.opened, self.duration(), self.loglevel
340
+ self.tag,
341
+ self.started,
342
+ self.opened,
343
+ self.duration(),
344
+ self.loglevel,
312
345
  )
313
346
  return card
314
347
 
@@ -333,7 +366,7 @@ class Ticket(getbytag.GetByTag):
333
366
  obj.catch_focus()
334
367
  return obj
335
368
  else:
336
- logger.error('Try to switch to an undefined session: %s', tag)
369
+ logger.error("Try to switch to an undefined session: %s", tag)
337
370
  return None
338
371
 
339
372
  def __del__(self):
vortex/syntax/__init__.py CHANGED
@@ -6,4 +6,4 @@ The most important usage is done by :class:`FootprintBase` derivated objects.
6
6
  #: No automatic export
7
7
  __all__ = []
8
8
 
9
- __tocinfoline__ = 'VORTEX package where standard attributes are defined.'
9
+ __tocinfoline__ = "VORTEX package where standard attributes are defined."