vortex-nwp 2.0.0b1__py3-none-any.whl → 2.0.0b2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (139) hide show
  1. vortex/__init__.py +59 -45
  2. vortex/algo/__init__.py +3 -2
  3. vortex/algo/components.py +940 -614
  4. vortex/algo/mpitools.py +802 -497
  5. vortex/algo/serversynctools.py +34 -33
  6. vortex/config.py +19 -22
  7. vortex/data/__init__.py +9 -3
  8. vortex/data/abstractstores.py +593 -655
  9. vortex/data/containers.py +217 -162
  10. vortex/data/contents.py +65 -39
  11. vortex/data/executables.py +93 -102
  12. vortex/data/flow.py +40 -34
  13. vortex/data/geometries.py +228 -132
  14. vortex/data/handlers.py +428 -225
  15. vortex/data/outflow.py +15 -15
  16. vortex/data/providers.py +185 -163
  17. vortex/data/resources.py +48 -42
  18. vortex/data/stores.py +544 -413
  19. vortex/gloves.py +114 -87
  20. vortex/layout/__init__.py +1 -8
  21. vortex/layout/contexts.py +150 -84
  22. vortex/layout/dataflow.py +353 -202
  23. vortex/layout/monitor.py +264 -128
  24. vortex/nwp/__init__.py +5 -2
  25. vortex/nwp/algo/__init__.py +14 -5
  26. vortex/nwp/algo/assim.py +205 -151
  27. vortex/nwp/algo/clim.py +683 -517
  28. vortex/nwp/algo/coupling.py +447 -225
  29. vortex/nwp/algo/eda.py +437 -229
  30. vortex/nwp/algo/eps.py +403 -231
  31. vortex/nwp/algo/forecasts.py +420 -271
  32. vortex/nwp/algo/fpserver.py +683 -307
  33. vortex/nwp/algo/ifsnaming.py +205 -145
  34. vortex/nwp/algo/ifsroot.py +210 -122
  35. vortex/nwp/algo/monitoring.py +132 -76
  36. vortex/nwp/algo/mpitools.py +321 -191
  37. vortex/nwp/algo/odbtools.py +617 -353
  38. vortex/nwp/algo/oopsroot.py +449 -273
  39. vortex/nwp/algo/oopstests.py +90 -56
  40. vortex/nwp/algo/request.py +287 -206
  41. vortex/nwp/algo/stdpost.py +878 -522
  42. vortex/nwp/data/__init__.py +22 -4
  43. vortex/nwp/data/assim.py +125 -137
  44. vortex/nwp/data/boundaries.py +121 -68
  45. vortex/nwp/data/climfiles.py +193 -211
  46. vortex/nwp/data/configfiles.py +73 -69
  47. vortex/nwp/data/consts.py +426 -401
  48. vortex/nwp/data/ctpini.py +59 -43
  49. vortex/nwp/data/diagnostics.py +94 -66
  50. vortex/nwp/data/eda.py +50 -51
  51. vortex/nwp/data/eps.py +195 -146
  52. vortex/nwp/data/executables.py +440 -434
  53. vortex/nwp/data/fields.py +63 -48
  54. vortex/nwp/data/gridfiles.py +183 -111
  55. vortex/nwp/data/logs.py +250 -217
  56. vortex/nwp/data/modelstates.py +180 -151
  57. vortex/nwp/data/monitoring.py +72 -99
  58. vortex/nwp/data/namelists.py +254 -202
  59. vortex/nwp/data/obs.py +400 -308
  60. vortex/nwp/data/oopsexec.py +22 -20
  61. vortex/nwp/data/providers.py +90 -65
  62. vortex/nwp/data/query.py +71 -82
  63. vortex/nwp/data/stores.py +49 -36
  64. vortex/nwp/data/surfex.py +136 -137
  65. vortex/nwp/syntax/__init__.py +1 -1
  66. vortex/nwp/syntax/stdattrs.py +173 -111
  67. vortex/nwp/tools/__init__.py +2 -2
  68. vortex/nwp/tools/addons.py +22 -17
  69. vortex/nwp/tools/agt.py +24 -12
  70. vortex/nwp/tools/bdap.py +16 -5
  71. vortex/nwp/tools/bdcp.py +4 -1
  72. vortex/nwp/tools/bdm.py +3 -0
  73. vortex/nwp/tools/bdmp.py +14 -9
  74. vortex/nwp/tools/conftools.py +728 -378
  75. vortex/nwp/tools/drhook.py +12 -8
  76. vortex/nwp/tools/grib.py +65 -39
  77. vortex/nwp/tools/gribdiff.py +22 -17
  78. vortex/nwp/tools/ifstools.py +82 -42
  79. vortex/nwp/tools/igastuff.py +167 -143
  80. vortex/nwp/tools/mars.py +14 -2
  81. vortex/nwp/tools/odb.py +234 -125
  82. vortex/nwp/tools/partitioning.py +61 -37
  83. vortex/nwp/tools/satrad.py +27 -12
  84. vortex/nwp/util/async.py +83 -55
  85. vortex/nwp/util/beacon.py +10 -10
  86. vortex/nwp/util/diffpygram.py +174 -86
  87. vortex/nwp/util/ens.py +144 -63
  88. vortex/nwp/util/hooks.py +30 -19
  89. vortex/nwp/util/taskdeco.py +28 -24
  90. vortex/nwp/util/usepygram.py +278 -172
  91. vortex/nwp/util/usetnt.py +31 -17
  92. vortex/sessions.py +72 -39
  93. vortex/syntax/__init__.py +1 -1
  94. vortex/syntax/stdattrs.py +410 -171
  95. vortex/syntax/stddeco.py +31 -22
  96. vortex/toolbox.py +327 -192
  97. vortex/tools/__init__.py +11 -2
  98. vortex/tools/actions.py +125 -59
  99. vortex/tools/addons.py +111 -92
  100. vortex/tools/arm.py +42 -22
  101. vortex/tools/compression.py +72 -69
  102. vortex/tools/date.py +11 -4
  103. vortex/tools/delayedactions.py +242 -132
  104. vortex/tools/env.py +75 -47
  105. vortex/tools/folder.py +342 -171
  106. vortex/tools/grib.py +311 -149
  107. vortex/tools/lfi.py +423 -216
  108. vortex/tools/listings.py +109 -40
  109. vortex/tools/names.py +218 -156
  110. vortex/tools/net.py +632 -298
  111. vortex/tools/parallelism.py +93 -61
  112. vortex/tools/prestaging.py +55 -31
  113. vortex/tools/schedulers.py +172 -105
  114. vortex/tools/services.py +402 -333
  115. vortex/tools/storage.py +293 -358
  116. vortex/tools/surfex.py +24 -24
  117. vortex/tools/systems.py +1211 -631
  118. vortex/tools/targets.py +156 -100
  119. vortex/util/__init__.py +1 -1
  120. vortex/util/config.py +377 -327
  121. vortex/util/empty.py +2 -2
  122. vortex/util/helpers.py +56 -24
  123. vortex/util/introspection.py +18 -12
  124. vortex/util/iosponge.py +8 -4
  125. vortex/util/roles.py +4 -6
  126. vortex/util/storefunctions.py +39 -13
  127. vortex/util/structs.py +3 -3
  128. vortex/util/worker.py +29 -17
  129. vortex_nwp-2.0.0b2.dist-info/METADATA +66 -0
  130. vortex_nwp-2.0.0b2.dist-info/RECORD +142 -0
  131. {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.0.0b2.dist-info}/WHEEL +1 -1
  132. vortex/layout/appconf.py +0 -109
  133. vortex/layout/jobs.py +0 -1276
  134. vortex/layout/nodes.py +0 -1424
  135. vortex/layout/subjobs.py +0 -464
  136. vortex_nwp-2.0.0b1.dist-info/METADATA +0 -50
  137. vortex_nwp-2.0.0b1.dist-info/RECORD +0 -146
  138. {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.0.0b2.dist-info}/LICENSE +0 -0
  139. {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.0.0b2.dist-info}/top_level.txt +0 -0
@@ -8,7 +8,11 @@ from bronx.stdtypes import date
8
8
  import footprints
9
9
  from vortex.tools import env
10
10
  from vortex.data.flow import GeoFlowResource, GeoPeriodFlowResource
11
- from vortex.syntax.stddeco import namebuilding_append, namebuilding_insert, overwrite_realkind
11
+ from vortex.syntax.stddeco import (
12
+ namebuilding_append,
13
+ namebuilding_insert,
14
+ overwrite_realkind,
15
+ )
12
16
  from vortex.syntax.stdattrs import term_deco, timeperiod_deco, a_cutoff
13
17
  from vortex.data.geometries import LonlatGeometry
14
18
 
@@ -18,8 +22,8 @@ from ..tools.igastuff import archive_suffix
18
22
  __all__ = []
19
23
 
20
24
 
21
- @namebuilding_insert('radical', lambda s: 'cpl')
22
- @namebuilding_insert('src', lambda s: s._mysrc)
25
+ @namebuilding_insert("radical", lambda s: "cpl")
26
+ @namebuilding_insert("src", lambda s: s._mysrc)
23
27
  class _AbstractLAMBoundary(GeoFlowResource):
24
28
  """
25
29
  Class of a coupling file for a Limited Area Model.
@@ -30,23 +34,30 @@ class _AbstractLAMBoundary(GeoFlowResource):
30
34
  _footprint = [
31
35
  term_deco,
32
36
  dict(
33
- info = 'Coupling file for a limited area model',
34
- attr = dict(
35
- kind = dict(
36
- values = ['boundary', 'elscf', 'coupled'],
37
- remap = dict(autoremap = 'first'),
37
+ info="Coupling file for a limited area model",
38
+ attr=dict(
39
+ kind=dict(
40
+ values=["boundary", "elscf", "coupled"],
41
+ remap=dict(autoremap="first"),
38
42
  ),
39
- nativefmt = dict(
40
- values = ['fa', 'grib', 'netcdf', 'ascii', 'wbcpack', 'unknown'],
41
- default = 'fa',
43
+ nativefmt=dict(
44
+ values=[
45
+ "fa",
46
+ "grib",
47
+ "netcdf",
48
+ "ascii",
49
+ "wbcpack",
50
+ "unknown",
51
+ ],
52
+ default="fa",
42
53
  ),
43
- )
44
- )
54
+ ),
55
+ ),
45
56
  ]
46
57
 
47
58
  @property
48
59
  def realkind(self):
49
- return 'boundary'
60
+ return "boundary"
50
61
 
51
62
  @property
52
63
  def _mysrc(self):
@@ -54,41 +65,43 @@ class _AbstractLAMBoundary(GeoFlowResource):
54
65
 
55
66
  def olive_basename(self):
56
67
  """OLIVE specific naming convention."""
57
- if self.mailbox.get('block', '-') == 'surfan':
68
+ if self.mailbox.get("block", "-") == "surfan":
58
69
  hhreal = self.term
59
70
  else:
60
71
  e = env.current()
61
- if 'HHDELTA_CPL' in e:
62
- actualbase = self.date - date.Time(e.HHDELTA_CPL + 'H')
72
+ if "HHDELTA_CPL" in e:
73
+ actualbase = self.date - date.Time(e.HHDELTA_CPL + "H")
63
74
  else:
64
75
  actualbase = date.synop(base=self.date)
65
76
  hhreal = (self.date - actualbase).time() + self.term
66
- return 'ELSCFALAD_' + self.geometry.area + '+' + hhreal.fmthour
77
+ return "ELSCFALAD_" + self.geometry.area + "+" + hhreal.fmthour
67
78
 
68
79
  def archive_basename(self):
69
80
  """OP ARCHIVE specific naming convention."""
70
81
  suffix = archive_suffix(self.model, self.cutoff, self.date)
71
- prefix = 'COUPL'
72
- source = self._mysrc[0] if isinstance(self._mysrc, list) else self._mysrc
73
- if re.match('assist1bis|testms1', self.geometry.area):
74
- prefix = 'COUPL1'
75
- if re.match('ifs|ecmwf', source) and '16km' in self.geometry.rnice:
76
- prefix = 'COUPLIFS'
77
-
78
- if self.model == 'mocage':
82
+ prefix = "COUPL"
83
+ source = (
84
+ self._mysrc[0] if isinstance(self._mysrc, list) else self._mysrc
85
+ )
86
+ if re.match("assist1bis|testms1", self.geometry.area):
87
+ prefix = "COUPL1"
88
+ if re.match("ifs|ecmwf", source) and "16km" in self.geometry.rnice:
89
+ prefix = "COUPLIFS"
90
+
91
+ if self.model == "mocage":
79
92
  valid = (self.date + self.term).ymd
80
- return 'SM' + self.geometry.area + '+' + valid
93
+ return "SM" + self.geometry.area + "+" + valid
81
94
 
82
- return prefix + self.term.fmthour + '.r{!s}'.format(suffix)
95
+ return prefix + self.term.fmthour + ".r{!s}".format(suffix)
83
96
 
84
97
  def iga_pathinfo(self):
85
98
  """Standard path information for IGA inline cache."""
86
- if self.model == 'arome':
87
- directory = 'fic_day'
88
- elif self.model == 'mfwam':
89
- directory = 'guess'
99
+ if self.model == "arome":
100
+ directory = "fic_day"
101
+ elif self.model == "mfwam":
102
+ directory = "guess"
90
103
  else:
91
- directory = 'autres'
104
+ directory = "autres"
92
105
  return dict(
93
106
  fmt=directory,
94
107
  model=self.model,
@@ -111,11 +124,18 @@ class LAMBoundary(_AbstractLAMBoundary):
111
124
  """
112
125
 
113
126
  _footprint = dict(
114
- attr = dict(
115
- source = dict(
116
- values = [
117
- 'arpege', 'aladin', 'arome', 'ifs', 'ecmwf', 'psy4',
118
- 'mercator_global', 'glo12', 'mfwam'
127
+ attr=dict(
128
+ source=dict(
129
+ values=[
130
+ "arpege",
131
+ "aladin",
132
+ "arome",
133
+ "ifs",
134
+ "ecmwf",
135
+ "psy4",
136
+ "mercator_global",
137
+ "glo12",
138
+ "mfwam",
119
139
  ]
120
140
  ),
121
141
  )
@@ -127,9 +147,9 @@ class LAMBoundary(_AbstractLAMBoundary):
127
147
 
128
148
 
129
149
  _a_source_cutoff = a_cutoff
130
- del _a_source_cutoff['alias']
131
- _a_source_cutoff['optional'] = True
132
- _a_source_cutoff['default'] = 'production'
150
+ del _a_source_cutoff["alias"]
151
+ _a_source_cutoff["optional"] = True
152
+ _a_source_cutoff["default"] = "production"
133
153
 
134
154
 
135
155
  class EnhancedLAMBoundary(_AbstractLAMBoundary):
@@ -140,33 +160,46 @@ class EnhancedLAMBoundary(_AbstractLAMBoundary):
140
160
  """
141
161
 
142
162
  _footprint = dict(
143
- attr = dict(
144
- source_app = dict(),
145
- source_conf = dict(),
146
- source_cutoff = _a_source_cutoff,
163
+ attr=dict(
164
+ source_app=dict(),
165
+ source_conf=dict(),
166
+ source_cutoff=_a_source_cutoff,
147
167
  )
148
168
  )
149
169
 
150
170
  @property
151
171
  def _mysrc(self):
152
- return [self.source_app, self.source_conf,
153
- {'cutoff': self.source_cutoff}]
172
+ return [
173
+ self.source_app,
174
+ self.source_conf,
175
+ {"cutoff": self.source_cutoff},
176
+ ]
154
177
 
155
178
 
156
179
  _abs_forcing_fp = footprints.DecorativeFootprint(
157
- info='Coupling file for any offline model.',
180
+ info="Coupling file for any offline model.",
158
181
  attr=dict(
159
182
  kind=dict(
160
- values=['forcing', ],
183
+ values=[
184
+ "forcing",
185
+ ],
161
186
  ),
162
187
  filling=dict(),
163
188
  source_app=dict(),
164
189
  source_conf=dict(),
165
190
  source_cutoff=_a_source_cutoff,
166
191
  ),
167
- decorator=[namebuilding_insert('src', lambda s: [s.source_app, s.source_conf,
168
- {'cutoff': s.source_cutoff}]),
169
- overwrite_realkind('forcing'), ]
192
+ decorator=[
193
+ namebuilding_insert(
194
+ "src",
195
+ lambda s: [
196
+ s.source_app,
197
+ s.source_conf,
198
+ {"cutoff": s.source_cutoff},
199
+ ],
200
+ ),
201
+ overwrite_realkind("forcing"),
202
+ ],
170
203
  )
171
204
 
172
205
 
@@ -174,25 +207,31 @@ class _AbstractForcing(GeoFlowResource):
174
207
  """Abstract class for date-based coupling file for any offline model."""
175
208
 
176
209
  _abstract = True
177
- _footprint = [_abs_forcing_fp, ]
210
+ _footprint = [
211
+ _abs_forcing_fp,
212
+ ]
178
213
 
179
214
 
180
215
  class _AbstractPeriodForcing(GeoPeriodFlowResource):
181
216
  """Abstract class for period-based coupling file for any offline model."""
182
217
 
183
218
  _abstract = True
184
- _footprint = [_abs_forcing_fp, ]
219
+ _footprint = [
220
+ _abs_forcing_fp,
221
+ ]
185
222
 
186
223
 
187
224
  _abs_external_forcing_fp = footprints.DecorativeFootprint(
188
225
  dict(
189
226
  attr=dict(
190
227
  model=dict(
191
- outcast=['surfex', ],
228
+ outcast=[
229
+ "surfex",
230
+ ],
192
231
  ),
193
232
  ),
194
233
  ),
195
- decorator=[namebuilding_append('src', lambda s: s.filling)],
234
+ decorator=[namebuilding_append("src", lambda s: s.filling)],
196
235
  )
197
236
 
198
237
 
@@ -216,24 +255,30 @@ class ExternalTimePeriodForcing(_AbstractForcing):
216
255
 
217
256
  _abs_surfex_forcing_fp = footprints.DecorativeFootprint(
218
257
  dict(
219
- info='Coupling/Forcing file for Surfex.',
258
+ info="Coupling/Forcing file for Surfex.",
220
259
  attr=dict(
221
260
  model=dict(
222
- values=['surfex', ],
261
+ values=[
262
+ "surfex",
263
+ ],
223
264
  ),
224
265
  filling=dict(
225
266
  optional=True,
226
- default='atm',
267
+ default="atm",
227
268
  ),
228
269
  nativefmt=dict(
229
- values=['netcdf', 'ascii', 'tar'],
230
- default='netcdf',
270
+ values=["netcdf", "ascii", "tar"],
271
+ default="netcdf",
231
272
  ),
232
- )
273
+ ),
233
274
  ),
234
- decorator=[namebuilding_append('src',
235
- lambda s: None if s.filling == 'atm' else s.filling,
236
- none_discard=True)]
275
+ decorator=[
276
+ namebuilding_append(
277
+ "src",
278
+ lambda s: None if s.filling == "atm" else s.filling,
279
+ none_discard=True,
280
+ )
281
+ ],
237
282
  )
238
283
 
239
284
 
@@ -243,7 +288,10 @@ class SurfexForcing(_AbstractForcing):
243
288
  This class takes an optional **term** attribute.
244
289
  """
245
290
 
246
- _footprint = [term_deco, _abs_surfex_forcing_fp, ]
291
+ _footprint = [
292
+ term_deco,
293
+ _abs_surfex_forcing_fp,
294
+ ]
247
295
 
248
296
 
249
297
  class SurfexTimePeriodForcing(_AbstractForcing):
@@ -252,10 +300,15 @@ class SurfexTimePeriodForcing(_AbstractForcing):
252
300
  This class needs a **begintime**/**endtime** attribute.
253
301
  """
254
302
 
255
- _footprint = [timeperiod_deco, _abs_surfex_forcing_fp, ]
303
+ _footprint = [
304
+ timeperiod_deco,
305
+ _abs_surfex_forcing_fp,
306
+ ]
256
307
 
257
308
 
258
309
  class SurfexPeriodForcing(_AbstractPeriodForcing):
259
310
  """Class for period-based coupling file for Surfex."""
260
311
 
261
- _footprint = [_abs_surfex_forcing_fp, ]
312
+ _footprint = [
313
+ _abs_surfex_forcing_fp,
314
+ ]