vortex-nwp 2.0.0b1__py3-none-any.whl → 2.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (141) hide show
  1. vortex/__init__.py +75 -47
  2. vortex/algo/__init__.py +3 -2
  3. vortex/algo/components.py +944 -618
  4. vortex/algo/mpitools.py +802 -497
  5. vortex/algo/mpitools_templates/__init__.py +1 -0
  6. vortex/algo/serversynctools.py +34 -33
  7. vortex/config.py +19 -22
  8. vortex/data/__init__.py +9 -3
  9. vortex/data/abstractstores.py +593 -655
  10. vortex/data/containers.py +217 -162
  11. vortex/data/contents.py +65 -39
  12. vortex/data/executables.py +93 -102
  13. vortex/data/flow.py +40 -34
  14. vortex/data/geometries.py +228 -132
  15. vortex/data/handlers.py +436 -227
  16. vortex/data/outflow.py +15 -15
  17. vortex/data/providers.py +185 -163
  18. vortex/data/resources.py +48 -42
  19. vortex/data/stores.py +540 -417
  20. vortex/data/sync_templates/__init__.py +0 -0
  21. vortex/gloves.py +114 -87
  22. vortex/layout/__init__.py +1 -8
  23. vortex/layout/contexts.py +150 -84
  24. vortex/layout/dataflow.py +353 -202
  25. vortex/layout/monitor.py +264 -128
  26. vortex/nwp/__init__.py +5 -2
  27. vortex/nwp/algo/__init__.py +14 -5
  28. vortex/nwp/algo/assim.py +205 -151
  29. vortex/nwp/algo/clim.py +683 -517
  30. vortex/nwp/algo/coupling.py +447 -225
  31. vortex/nwp/algo/eda.py +437 -229
  32. vortex/nwp/algo/eps.py +403 -231
  33. vortex/nwp/algo/forecasts.py +416 -275
  34. vortex/nwp/algo/fpserver.py +683 -307
  35. vortex/nwp/algo/ifsnaming.py +205 -145
  36. vortex/nwp/algo/ifsroot.py +215 -122
  37. vortex/nwp/algo/monitoring.py +137 -76
  38. vortex/nwp/algo/mpitools.py +330 -190
  39. vortex/nwp/algo/odbtools.py +637 -353
  40. vortex/nwp/algo/oopsroot.py +454 -273
  41. vortex/nwp/algo/oopstests.py +90 -56
  42. vortex/nwp/algo/request.py +287 -206
  43. vortex/nwp/algo/stdpost.py +878 -522
  44. vortex/nwp/data/__init__.py +22 -4
  45. vortex/nwp/data/assim.py +125 -137
  46. vortex/nwp/data/boundaries.py +121 -68
  47. vortex/nwp/data/climfiles.py +193 -211
  48. vortex/nwp/data/configfiles.py +73 -69
  49. vortex/nwp/data/consts.py +426 -401
  50. vortex/nwp/data/ctpini.py +59 -43
  51. vortex/nwp/data/diagnostics.py +94 -66
  52. vortex/nwp/data/eda.py +50 -51
  53. vortex/nwp/data/eps.py +195 -146
  54. vortex/nwp/data/executables.py +440 -434
  55. vortex/nwp/data/fields.py +63 -48
  56. vortex/nwp/data/gridfiles.py +183 -111
  57. vortex/nwp/data/logs.py +250 -217
  58. vortex/nwp/data/modelstates.py +180 -151
  59. vortex/nwp/data/monitoring.py +72 -99
  60. vortex/nwp/data/namelists.py +254 -202
  61. vortex/nwp/data/obs.py +400 -308
  62. vortex/nwp/data/oopsexec.py +22 -20
  63. vortex/nwp/data/providers.py +90 -65
  64. vortex/nwp/data/query.py +71 -82
  65. vortex/nwp/data/stores.py +49 -36
  66. vortex/nwp/data/surfex.py +136 -137
  67. vortex/nwp/syntax/__init__.py +1 -1
  68. vortex/nwp/syntax/stdattrs.py +173 -111
  69. vortex/nwp/tools/__init__.py +2 -2
  70. vortex/nwp/tools/addons.py +22 -17
  71. vortex/nwp/tools/agt.py +24 -12
  72. vortex/nwp/tools/bdap.py +16 -5
  73. vortex/nwp/tools/bdcp.py +4 -1
  74. vortex/nwp/tools/bdm.py +3 -0
  75. vortex/nwp/tools/bdmp.py +14 -9
  76. vortex/nwp/tools/conftools.py +728 -378
  77. vortex/nwp/tools/drhook.py +12 -8
  78. vortex/nwp/tools/grib.py +65 -39
  79. vortex/nwp/tools/gribdiff.py +22 -17
  80. vortex/nwp/tools/ifstools.py +82 -42
  81. vortex/nwp/tools/igastuff.py +167 -143
  82. vortex/nwp/tools/mars.py +14 -2
  83. vortex/nwp/tools/odb.py +234 -125
  84. vortex/nwp/tools/partitioning.py +61 -37
  85. vortex/nwp/tools/satrad.py +27 -12
  86. vortex/nwp/util/async.py +83 -55
  87. vortex/nwp/util/beacon.py +10 -10
  88. vortex/nwp/util/diffpygram.py +174 -86
  89. vortex/nwp/util/ens.py +144 -63
  90. vortex/nwp/util/hooks.py +30 -19
  91. vortex/nwp/util/taskdeco.py +28 -24
  92. vortex/nwp/util/usepygram.py +278 -172
  93. vortex/nwp/util/usetnt.py +31 -17
  94. vortex/sessions.py +72 -39
  95. vortex/syntax/__init__.py +1 -1
  96. vortex/syntax/stdattrs.py +410 -171
  97. vortex/syntax/stddeco.py +31 -22
  98. vortex/toolbox.py +327 -192
  99. vortex/tools/__init__.py +11 -2
  100. vortex/tools/actions.py +110 -121
  101. vortex/tools/addons.py +111 -92
  102. vortex/tools/arm.py +42 -22
  103. vortex/tools/compression.py +72 -69
  104. vortex/tools/date.py +11 -4
  105. vortex/tools/delayedactions.py +242 -132
  106. vortex/tools/env.py +75 -47
  107. vortex/tools/folder.py +342 -171
  108. vortex/tools/grib.py +341 -162
  109. vortex/tools/lfi.py +423 -216
  110. vortex/tools/listings.py +109 -40
  111. vortex/tools/names.py +218 -156
  112. vortex/tools/net.py +655 -299
  113. vortex/tools/parallelism.py +93 -61
  114. vortex/tools/prestaging.py +55 -31
  115. vortex/tools/schedulers.py +172 -105
  116. vortex/tools/services.py +403 -334
  117. vortex/tools/storage.py +293 -358
  118. vortex/tools/surfex.py +24 -24
  119. vortex/tools/systems.py +1234 -643
  120. vortex/tools/targets.py +156 -100
  121. vortex/util/__init__.py +1 -1
  122. vortex/util/config.py +378 -327
  123. vortex/util/empty.py +2 -2
  124. vortex/util/helpers.py +56 -24
  125. vortex/util/introspection.py +18 -12
  126. vortex/util/iosponge.py +8 -4
  127. vortex/util/roles.py +4 -6
  128. vortex/util/storefunctions.py +39 -13
  129. vortex/util/structs.py +3 -3
  130. vortex/util/worker.py +29 -17
  131. vortex_nwp-2.1.0.dist-info/METADATA +67 -0
  132. vortex_nwp-2.1.0.dist-info/RECORD +144 -0
  133. {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.1.0.dist-info}/WHEEL +1 -1
  134. vortex/layout/appconf.py +0 -109
  135. vortex/layout/jobs.py +0 -1276
  136. vortex/layout/nodes.py +0 -1424
  137. vortex/layout/subjobs.py +0 -464
  138. vortex_nwp-2.0.0b1.dist-info/METADATA +0 -50
  139. vortex_nwp-2.0.0b1.dist-info/RECORD +0 -146
  140. {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.1.0.dist-info/licenses}/LICENSE +0 -0
  141. {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.1.0.dist-info}/top_level.txt +0 -0
@@ -8,7 +8,11 @@ from bronx.stdtypes import date
8
8
  import footprints
9
9
  from vortex.tools import env
10
10
  from vortex.data.flow import GeoFlowResource, GeoPeriodFlowResource
11
- from vortex.syntax.stddeco import namebuilding_append, namebuilding_insert, overwrite_realkind
11
+ from vortex.syntax.stddeco import (
12
+ namebuilding_append,
13
+ namebuilding_insert,
14
+ overwrite_realkind,
15
+ )
12
16
  from vortex.syntax.stdattrs import term_deco, timeperiod_deco, a_cutoff
13
17
  from vortex.data.geometries import LonlatGeometry
14
18
 
@@ -18,8 +22,8 @@ from ..tools.igastuff import archive_suffix
18
22
  __all__ = []
19
23
 
20
24
 
21
- @namebuilding_insert('radical', lambda s: 'cpl')
22
- @namebuilding_insert('src', lambda s: s._mysrc)
25
+ @namebuilding_insert("radical", lambda s: "cpl")
26
+ @namebuilding_insert("src", lambda s: s._mysrc)
23
27
  class _AbstractLAMBoundary(GeoFlowResource):
24
28
  """
25
29
  Class of a coupling file for a Limited Area Model.
@@ -30,23 +34,30 @@ class _AbstractLAMBoundary(GeoFlowResource):
30
34
  _footprint = [
31
35
  term_deco,
32
36
  dict(
33
- info = 'Coupling file for a limited area model',
34
- attr = dict(
35
- kind = dict(
36
- values = ['boundary', 'elscf', 'coupled'],
37
- remap = dict(autoremap = 'first'),
37
+ info="Coupling file for a limited area model",
38
+ attr=dict(
39
+ kind=dict(
40
+ values=["boundary", "elscf", "coupled"],
41
+ remap=dict(autoremap="first"),
38
42
  ),
39
- nativefmt = dict(
40
- values = ['fa', 'grib', 'netcdf', 'ascii', 'wbcpack', 'unknown'],
41
- default = 'fa',
43
+ nativefmt=dict(
44
+ values=[
45
+ "fa",
46
+ "grib",
47
+ "netcdf",
48
+ "ascii",
49
+ "wbcpack",
50
+ "unknown",
51
+ ],
52
+ default="fa",
42
53
  ),
43
- )
44
- )
54
+ ),
55
+ ),
45
56
  ]
46
57
 
47
58
  @property
48
59
  def realkind(self):
49
- return 'boundary'
60
+ return "boundary"
50
61
 
51
62
  @property
52
63
  def _mysrc(self):
@@ -54,41 +65,43 @@ class _AbstractLAMBoundary(GeoFlowResource):
54
65
 
55
66
  def olive_basename(self):
56
67
  """OLIVE specific naming convention."""
57
- if self.mailbox.get('block', '-') == 'surfan':
68
+ if self.mailbox.get("block", "-") == "surfan":
58
69
  hhreal = self.term
59
70
  else:
60
71
  e = env.current()
61
- if 'HHDELTA_CPL' in e:
62
- actualbase = self.date - date.Time(e.HHDELTA_CPL + 'H')
72
+ if "HHDELTA_CPL" in e:
73
+ actualbase = self.date - date.Time(e.HHDELTA_CPL + "H")
63
74
  else:
64
75
  actualbase = date.synop(base=self.date)
65
76
  hhreal = (self.date - actualbase).time() + self.term
66
- return 'ELSCFALAD_' + self.geometry.area + '+' + hhreal.fmthour
77
+ return "ELSCFALAD_" + self.geometry.area + "+" + hhreal.fmthour
67
78
 
68
79
  def archive_basename(self):
69
80
  """OP ARCHIVE specific naming convention."""
70
81
  suffix = archive_suffix(self.model, self.cutoff, self.date)
71
- prefix = 'COUPL'
72
- source = self._mysrc[0] if isinstance(self._mysrc, list) else self._mysrc
73
- if re.match('assist1bis|testms1', self.geometry.area):
74
- prefix = 'COUPL1'
75
- if re.match('ifs|ecmwf', source) and '16km' in self.geometry.rnice:
76
- prefix = 'COUPLIFS'
77
-
78
- if self.model == 'mocage':
82
+ prefix = "COUPL"
83
+ source = (
84
+ self._mysrc[0] if isinstance(self._mysrc, list) else self._mysrc
85
+ )
86
+ if re.match("assist1bis|testms1", self.geometry.area):
87
+ prefix = "COUPL1"
88
+ if re.match("ifs|ecmwf", source) and "16km" in self.geometry.rnice:
89
+ prefix = "COUPLIFS"
90
+
91
+ if self.model == "mocage":
79
92
  valid = (self.date + self.term).ymd
80
- return 'SM' + self.geometry.area + '+' + valid
93
+ return "SM" + self.geometry.area + "+" + valid
81
94
 
82
- return prefix + self.term.fmthour + '.r{!s}'.format(suffix)
95
+ return prefix + self.term.fmthour + ".r{!s}".format(suffix)
83
96
 
84
97
  def iga_pathinfo(self):
85
98
  """Standard path information for IGA inline cache."""
86
- if self.model == 'arome':
87
- directory = 'fic_day'
88
- elif self.model == 'mfwam':
89
- directory = 'guess'
99
+ if self.model == "arome":
100
+ directory = "fic_day"
101
+ elif self.model == "mfwam":
102
+ directory = "guess"
90
103
  else:
91
- directory = 'autres'
104
+ directory = "autres"
92
105
  return dict(
93
106
  fmt=directory,
94
107
  model=self.model,
@@ -111,11 +124,18 @@ class LAMBoundary(_AbstractLAMBoundary):
111
124
  """
112
125
 
113
126
  _footprint = dict(
114
- attr = dict(
115
- source = dict(
116
- values = [
117
- 'arpege', 'aladin', 'arome', 'ifs', 'ecmwf', 'psy4',
118
- 'mercator_global', 'glo12', 'mfwam'
127
+ attr=dict(
128
+ source=dict(
129
+ values=[
130
+ "arpege",
131
+ "aladin",
132
+ "arome",
133
+ "ifs",
134
+ "ecmwf",
135
+ "psy4",
136
+ "mercator_global",
137
+ "glo12",
138
+ "mfwam",
119
139
  ]
120
140
  ),
121
141
  )
@@ -127,9 +147,9 @@ class LAMBoundary(_AbstractLAMBoundary):
127
147
 
128
148
 
129
149
  _a_source_cutoff = a_cutoff
130
- del _a_source_cutoff['alias']
131
- _a_source_cutoff['optional'] = True
132
- _a_source_cutoff['default'] = 'production'
150
+ del _a_source_cutoff["alias"]
151
+ _a_source_cutoff["optional"] = True
152
+ _a_source_cutoff["default"] = "production"
133
153
 
134
154
 
135
155
  class EnhancedLAMBoundary(_AbstractLAMBoundary):
@@ -140,33 +160,46 @@ class EnhancedLAMBoundary(_AbstractLAMBoundary):
140
160
  """
141
161
 
142
162
  _footprint = dict(
143
- attr = dict(
144
- source_app = dict(),
145
- source_conf = dict(),
146
- source_cutoff = _a_source_cutoff,
163
+ attr=dict(
164
+ source_app=dict(),
165
+ source_conf=dict(),
166
+ source_cutoff=_a_source_cutoff,
147
167
  )
148
168
  )
149
169
 
150
170
  @property
151
171
  def _mysrc(self):
152
- return [self.source_app, self.source_conf,
153
- {'cutoff': self.source_cutoff}]
172
+ return [
173
+ self.source_app,
174
+ self.source_conf,
175
+ {"cutoff": self.source_cutoff},
176
+ ]
154
177
 
155
178
 
156
179
  _abs_forcing_fp = footprints.DecorativeFootprint(
157
- info='Coupling file for any offline model.',
180
+ info="Coupling file for any offline model.",
158
181
  attr=dict(
159
182
  kind=dict(
160
- values=['forcing', ],
183
+ values=[
184
+ "forcing",
185
+ ],
161
186
  ),
162
187
  filling=dict(),
163
188
  source_app=dict(),
164
189
  source_conf=dict(),
165
190
  source_cutoff=_a_source_cutoff,
166
191
  ),
167
- decorator=[namebuilding_insert('src', lambda s: [s.source_app, s.source_conf,
168
- {'cutoff': s.source_cutoff}]),
169
- overwrite_realkind('forcing'), ]
192
+ decorator=[
193
+ namebuilding_insert(
194
+ "src",
195
+ lambda s: [
196
+ s.source_app,
197
+ s.source_conf,
198
+ {"cutoff": s.source_cutoff},
199
+ ],
200
+ ),
201
+ overwrite_realkind("forcing"),
202
+ ],
170
203
  )
171
204
 
172
205
 
@@ -174,25 +207,31 @@ class _AbstractForcing(GeoFlowResource):
174
207
  """Abstract class for date-based coupling file for any offline model."""
175
208
 
176
209
  _abstract = True
177
- _footprint = [_abs_forcing_fp, ]
210
+ _footprint = [
211
+ _abs_forcing_fp,
212
+ ]
178
213
 
179
214
 
180
215
  class _AbstractPeriodForcing(GeoPeriodFlowResource):
181
216
  """Abstract class for period-based coupling file for any offline model."""
182
217
 
183
218
  _abstract = True
184
- _footprint = [_abs_forcing_fp, ]
219
+ _footprint = [
220
+ _abs_forcing_fp,
221
+ ]
185
222
 
186
223
 
187
224
  _abs_external_forcing_fp = footprints.DecorativeFootprint(
188
225
  dict(
189
226
  attr=dict(
190
227
  model=dict(
191
- outcast=['surfex', ],
228
+ outcast=[
229
+ "surfex",
230
+ ],
192
231
  ),
193
232
  ),
194
233
  ),
195
- decorator=[namebuilding_append('src', lambda s: s.filling)],
234
+ decorator=[namebuilding_append("src", lambda s: s.filling)],
196
235
  )
197
236
 
198
237
 
@@ -216,24 +255,30 @@ class ExternalTimePeriodForcing(_AbstractForcing):
216
255
 
217
256
  _abs_surfex_forcing_fp = footprints.DecorativeFootprint(
218
257
  dict(
219
- info='Coupling/Forcing file for Surfex.',
258
+ info="Coupling/Forcing file for Surfex.",
220
259
  attr=dict(
221
260
  model=dict(
222
- values=['surfex', ],
261
+ values=[
262
+ "surfex",
263
+ ],
223
264
  ),
224
265
  filling=dict(
225
266
  optional=True,
226
- default='atm',
267
+ default="atm",
227
268
  ),
228
269
  nativefmt=dict(
229
- values=['netcdf', 'ascii', 'tar'],
230
- default='netcdf',
270
+ values=["netcdf", "ascii", "tar"],
271
+ default="netcdf",
231
272
  ),
232
- )
273
+ ),
233
274
  ),
234
- decorator=[namebuilding_append('src',
235
- lambda s: None if s.filling == 'atm' else s.filling,
236
- none_discard=True)]
275
+ decorator=[
276
+ namebuilding_append(
277
+ "src",
278
+ lambda s: None if s.filling == "atm" else s.filling,
279
+ none_discard=True,
280
+ )
281
+ ],
237
282
  )
238
283
 
239
284
 
@@ -243,7 +288,10 @@ class SurfexForcing(_AbstractForcing):
243
288
  This class takes an optional **term** attribute.
244
289
  """
245
290
 
246
- _footprint = [term_deco, _abs_surfex_forcing_fp, ]
291
+ _footprint = [
292
+ term_deco,
293
+ _abs_surfex_forcing_fp,
294
+ ]
247
295
 
248
296
 
249
297
  class SurfexTimePeriodForcing(_AbstractForcing):
@@ -252,10 +300,15 @@ class SurfexTimePeriodForcing(_AbstractForcing):
252
300
  This class needs a **begintime**/**endtime** attribute.
253
301
  """
254
302
 
255
- _footprint = [timeperiod_deco, _abs_surfex_forcing_fp, ]
303
+ _footprint = [
304
+ timeperiod_deco,
305
+ _abs_surfex_forcing_fp,
306
+ ]
256
307
 
257
308
 
258
309
  class SurfexPeriodForcing(_AbstractPeriodForcing):
259
310
  """Class for period-based coupling file for Surfex."""
260
311
 
261
- _footprint = [_abs_surfex_forcing_fp, ]
312
+ _footprint = [
313
+ _abs_surfex_forcing_fp,
314
+ ]