vortex-nwp 2.0.0b1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (146) hide show
  1. vortex/__init__.py +135 -0
  2. vortex/algo/__init__.py +12 -0
  3. vortex/algo/components.py +2136 -0
  4. vortex/algo/mpitools.py +1648 -0
  5. vortex/algo/mpitools_templates/envelope_wrapper_default.tpl +27 -0
  6. vortex/algo/mpitools_templates/envelope_wrapper_mpiauto.tpl +29 -0
  7. vortex/algo/mpitools_templates/wrapstd_wrapper_default.tpl +18 -0
  8. vortex/algo/serversynctools.py +170 -0
  9. vortex/config.py +115 -0
  10. vortex/data/__init__.py +13 -0
  11. vortex/data/abstractstores.py +1572 -0
  12. vortex/data/containers.py +780 -0
  13. vortex/data/contents.py +596 -0
  14. vortex/data/executables.py +284 -0
  15. vortex/data/flow.py +113 -0
  16. vortex/data/geometries.ini +2689 -0
  17. vortex/data/geometries.py +703 -0
  18. vortex/data/handlers.py +1021 -0
  19. vortex/data/outflow.py +67 -0
  20. vortex/data/providers.py +465 -0
  21. vortex/data/resources.py +201 -0
  22. vortex/data/stores.py +1271 -0
  23. vortex/gloves.py +282 -0
  24. vortex/layout/__init__.py +27 -0
  25. vortex/layout/appconf.py +109 -0
  26. vortex/layout/contexts.py +511 -0
  27. vortex/layout/dataflow.py +1069 -0
  28. vortex/layout/jobs.py +1276 -0
  29. vortex/layout/monitor.py +833 -0
  30. vortex/layout/nodes.py +1424 -0
  31. vortex/layout/subjobs.py +464 -0
  32. vortex/nwp/__init__.py +11 -0
  33. vortex/nwp/algo/__init__.py +12 -0
  34. vortex/nwp/algo/assim.py +483 -0
  35. vortex/nwp/algo/clim.py +920 -0
  36. vortex/nwp/algo/coupling.py +609 -0
  37. vortex/nwp/algo/eda.py +632 -0
  38. vortex/nwp/algo/eps.py +613 -0
  39. vortex/nwp/algo/forecasts.py +745 -0
  40. vortex/nwp/algo/fpserver.py +927 -0
  41. vortex/nwp/algo/ifsnaming.py +403 -0
  42. vortex/nwp/algo/ifsroot.py +311 -0
  43. vortex/nwp/algo/monitoring.py +202 -0
  44. vortex/nwp/algo/mpitools.py +554 -0
  45. vortex/nwp/algo/odbtools.py +974 -0
  46. vortex/nwp/algo/oopsroot.py +735 -0
  47. vortex/nwp/algo/oopstests.py +186 -0
  48. vortex/nwp/algo/request.py +579 -0
  49. vortex/nwp/algo/stdpost.py +1285 -0
  50. vortex/nwp/data/__init__.py +12 -0
  51. vortex/nwp/data/assim.py +392 -0
  52. vortex/nwp/data/boundaries.py +261 -0
  53. vortex/nwp/data/climfiles.py +539 -0
  54. vortex/nwp/data/configfiles.py +149 -0
  55. vortex/nwp/data/consts.py +929 -0
  56. vortex/nwp/data/ctpini.py +133 -0
  57. vortex/nwp/data/diagnostics.py +181 -0
  58. vortex/nwp/data/eda.py +148 -0
  59. vortex/nwp/data/eps.py +383 -0
  60. vortex/nwp/data/executables.py +1039 -0
  61. vortex/nwp/data/fields.py +96 -0
  62. vortex/nwp/data/gridfiles.py +308 -0
  63. vortex/nwp/data/logs.py +551 -0
  64. vortex/nwp/data/modelstates.py +334 -0
  65. vortex/nwp/data/monitoring.py +220 -0
  66. vortex/nwp/data/namelists.py +644 -0
  67. vortex/nwp/data/obs.py +748 -0
  68. vortex/nwp/data/oopsexec.py +72 -0
  69. vortex/nwp/data/providers.py +182 -0
  70. vortex/nwp/data/query.py +217 -0
  71. vortex/nwp/data/stores.py +147 -0
  72. vortex/nwp/data/surfex.py +338 -0
  73. vortex/nwp/syntax/__init__.py +9 -0
  74. vortex/nwp/syntax/stdattrs.py +375 -0
  75. vortex/nwp/tools/__init__.py +10 -0
  76. vortex/nwp/tools/addons.py +35 -0
  77. vortex/nwp/tools/agt.py +55 -0
  78. vortex/nwp/tools/bdap.py +48 -0
  79. vortex/nwp/tools/bdcp.py +38 -0
  80. vortex/nwp/tools/bdm.py +21 -0
  81. vortex/nwp/tools/bdmp.py +49 -0
  82. vortex/nwp/tools/conftools.py +1311 -0
  83. vortex/nwp/tools/drhook.py +62 -0
  84. vortex/nwp/tools/grib.py +268 -0
  85. vortex/nwp/tools/gribdiff.py +99 -0
  86. vortex/nwp/tools/ifstools.py +163 -0
  87. vortex/nwp/tools/igastuff.py +249 -0
  88. vortex/nwp/tools/mars.py +56 -0
  89. vortex/nwp/tools/odb.py +548 -0
  90. vortex/nwp/tools/partitioning.py +234 -0
  91. vortex/nwp/tools/satrad.py +56 -0
  92. vortex/nwp/util/__init__.py +6 -0
  93. vortex/nwp/util/async.py +184 -0
  94. vortex/nwp/util/beacon.py +40 -0
  95. vortex/nwp/util/diffpygram.py +359 -0
  96. vortex/nwp/util/ens.py +198 -0
  97. vortex/nwp/util/hooks.py +128 -0
  98. vortex/nwp/util/taskdeco.py +81 -0
  99. vortex/nwp/util/usepygram.py +591 -0
  100. vortex/nwp/util/usetnt.py +87 -0
  101. vortex/proxy.py +6 -0
  102. vortex/sessions.py +341 -0
  103. vortex/syntax/__init__.py +9 -0
  104. vortex/syntax/stdattrs.py +628 -0
  105. vortex/syntax/stddeco.py +176 -0
  106. vortex/toolbox.py +982 -0
  107. vortex/tools/__init__.py +11 -0
  108. vortex/tools/actions.py +457 -0
  109. vortex/tools/addons.py +297 -0
  110. vortex/tools/arm.py +76 -0
  111. vortex/tools/compression.py +322 -0
  112. vortex/tools/date.py +20 -0
  113. vortex/tools/ddhpack.py +10 -0
  114. vortex/tools/delayedactions.py +672 -0
  115. vortex/tools/env.py +513 -0
  116. vortex/tools/folder.py +663 -0
  117. vortex/tools/grib.py +559 -0
  118. vortex/tools/lfi.py +746 -0
  119. vortex/tools/listings.py +354 -0
  120. vortex/tools/names.py +575 -0
  121. vortex/tools/net.py +1790 -0
  122. vortex/tools/odb.py +10 -0
  123. vortex/tools/parallelism.py +336 -0
  124. vortex/tools/prestaging.py +186 -0
  125. vortex/tools/rawfiles.py +10 -0
  126. vortex/tools/schedulers.py +413 -0
  127. vortex/tools/services.py +871 -0
  128. vortex/tools/storage.py +1061 -0
  129. vortex/tools/surfex.py +61 -0
  130. vortex/tools/systems.py +3396 -0
  131. vortex/tools/targets.py +384 -0
  132. vortex/util/__init__.py +9 -0
  133. vortex/util/config.py +1071 -0
  134. vortex/util/empty.py +24 -0
  135. vortex/util/helpers.py +184 -0
  136. vortex/util/introspection.py +63 -0
  137. vortex/util/iosponge.py +76 -0
  138. vortex/util/roles.py +51 -0
  139. vortex/util/storefunctions.py +103 -0
  140. vortex/util/structs.py +26 -0
  141. vortex/util/worker.py +150 -0
  142. vortex_nwp-2.0.0b1.dist-info/LICENSE +517 -0
  143. vortex_nwp-2.0.0b1.dist-info/METADATA +50 -0
  144. vortex_nwp-2.0.0b1.dist-info/RECORD +146 -0
  145. vortex_nwp-2.0.0b1.dist-info/WHEEL +5 -0
  146. vortex_nwp-2.0.0b1.dist-info/top_level.txt +1 -0
vortex/nwp/algo/eda.py ADDED
@@ -0,0 +1,632 @@
1
+ """
2
+ AlgoComponents dedicated to computations related to the Ensemble Data Assimilation
3
+ system.
4
+ """
5
+
6
+ import math
7
+ import re
8
+
9
+ from bronx.fancies import loggers
10
+ from bronx.stdtypes.date import Month, Time
11
+ import footprints
12
+
13
+ from vortex.algo.components import AlgoComponentError
14
+ from .ifsroot import IFSParallel
15
+
16
+ #: Automatic export off
17
+ __all__ = []
18
+
19
+ logger = loggers.getLogger(__name__)
20
+
21
+
22
+ class IFSEdaAbstractAlgo(IFSParallel):
23
+ """Base class for any EDA related task wrapped into an IFS/Arpege binary."""
24
+
25
+ _abstract = True
26
+ _footprint = dict(
27
+ info='Base class for any EDA related task',
28
+ attr=dict(
29
+ inputnaming = dict(
30
+ info = 'Prescribe your own naming template for input files.',
31
+ optional = True,
32
+ doc_visibility = footprints.doc.visibility.ADVANCED,
33
+ ),
34
+ )
35
+ )
36
+
37
+ def naming_convention(self, kind, rh, actualfmt=None, **kwargs):
38
+ """Take into account the *inputnaming* attribute."""
39
+ if kind == 'edainput':
40
+ return super().naming_convention(kind, rh,
41
+ actualfmt=actualfmt,
42
+ namingformat=self.inputnaming,
43
+ **kwargs)
44
+ else:
45
+ return super().naming_convention(kind, rh,
46
+ actualfmt=actualfmt,
47
+ **kwargs)
48
+
49
+
50
+ class IFSEdaEnsembleAbstractAlgo(IFSEdaAbstractAlgo):
51
+ """Base class for any EDA related task wrapped into an IFS/Arpege binary.
52
+
53
+ This extends the :class:`IFSEdaAbstractAlgo` with a *nbmember* attribute and
54
+ the ability to detect the input files and re-number them (in order to be able
55
+ to deal with missing members).
56
+ """
57
+
58
+ _INPUTS_ROLE = 'ModelState'
59
+
60
+ _abstract = True
61
+ _footprint = dict(
62
+ info='Base class for any EDA related task',
63
+ attr=dict(
64
+ nbmember = dict(
65
+ info = "The number of members to deal will (auto-detected if omitted)",
66
+ type = int,
67
+ optional = True,
68
+ ),
69
+ nbmin = dict(
70
+ info = "The minimum number of input members that is mandatory to go one",
71
+ type = int,
72
+ optional = True,
73
+ default = 2,
74
+ )
75
+ )
76
+ )
77
+
78
+ def __init__(self, *kargs, **kwargs):
79
+ super().__init__(*kargs, **kwargs)
80
+ self._actual_nbe = self.nbmember
81
+
82
+ @property
83
+ def actual_nbe(self):
84
+ """The effective number of members."""
85
+ return self._actual_nbe
86
+
87
+ @property
88
+ def actual_totalnumber(self):
89
+ """The total number of members (=! actual_nbe for lagged ensembles. see below)."""
90
+ return self.actual_nbe
91
+
92
+ @property
93
+ def actual_nbmin(self):
94
+ """The minimum number of effective members that are mandatory to go one."""
95
+ return self.nbmin
96
+
97
+ @staticmethod
98
+ def _members_sorting_key(s):
99
+ """Return the sorting key for the **s** section."""
100
+ member = getattr(s.rh.provider, 'member', None)
101
+ member = - math.inf if member is None else member
102
+ block = getattr(s.rh.provider, 'block', '')
103
+ filename = getattr(s.rh.container, 'basename', '')
104
+ return member, block, filename
105
+
106
+ def _members_effective_inputs(self):
107
+ """The list of effective sections representing input data."""
108
+ return sorted(self.context.sequence.effective_inputs(role=self._INPUTS_ROLE),
109
+ key=self._members_sorting_key)
110
+
111
+ def _members_all_inputs(self):
112
+ """The list of sections representing input data."""
113
+ return sorted(self.context.sequence.filtered_inputs(role=self._INPUTS_ROLE),
114
+ key=self._members_sorting_key)
115
+
116
+ def _check_members_list_numbering(self, rh, mlist, mformats):
117
+ """Check if, for the **mlist** members list, some renaming id needed."""
118
+ if len(set(mlist)) != len(mlist):
119
+ logger.warning("Some members are duplicated. That's very strange...")
120
+ if self.nbmember is None:
121
+ self.algoassert(len(mformats) <= 1, 'Mixed formats are not allowed !')
122
+ elif self.nbmember and len(mformats) > 1:
123
+ logger.info('%s have mixed formats... please correct that.', self._INPUTS_ROLE)
124
+ if mlist and self.nbmember is not None:
125
+ # Consistency check
126
+ if len(mlist) != self.nbmember:
127
+ logger.warning('Discrepancy between *nbmember* and effective input files...' +
128
+ ' sticking with *nbmember*')
129
+ else:
130
+ logger.info("The input files member numbers checks out !")
131
+ return False # Ok, apparently the user knows what she/he is doing
132
+ elif self.nbmember and not mlist:
133
+ return False # Ok, apparently the user knows what she/he is doing
134
+ elif mlist and self.nbmember is None:
135
+ innc = self.naming_convention(kind='edainput', variant=self.kind, rh=rh,
136
+ totalnumber=self.actual_totalnumber,
137
+ actualfmt=mformats.pop())
138
+ checkfiles = [m for m in range(1, len(mlist) + 1)
139
+ if self.system.path.exists(innc(number=m))]
140
+ if len(checkfiles) == len(mlist):
141
+ logger.info("The input files numbering checks out !")
142
+ return False # Ok, apparently the user knows what she/he is doing
143
+ elif len(checkfiles) == 0:
144
+ return True
145
+ else:
146
+ raise AlgoComponentError('Members renumbering is needed but some ' +
147
+ 'files are blocking the way !')
148
+ elif len(mlist) == 0 and self.nbmember is None:
149
+ raise AlgoComponentError('No input files where found !')
150
+
151
+ def modelstate_needs_renumbering(self, rh):
152
+ """Check if, for the **mlist** members list, some renaming id needed."""
153
+ eff_sections = self._members_effective_inputs()
154
+ eff_members = [sec.rh.provider.member for sec in eff_sections]
155
+ eff_formats = {sec.rh.container.actualfmt for sec in eff_sections}
156
+ if eff_members and self.nbmember is None:
157
+ self._actual_nbe = len(eff_members)
158
+ return self._check_members_list_numbering(rh, eff_members, eff_formats)
159
+
160
+ def modelstate_renumbering(self, rh, mlist):
161
+ """Actualy rename the effective inputs."""
162
+ eff_format = mlist[0].rh.container.actualfmt
163
+ innc = self.naming_convention(kind='edainput', variant=self.kind, rh=rh,
164
+ totalnumber=self.actual_totalnumber,
165
+ actualfmt=eff_format)
166
+ for i, s in enumerate(mlist, start=1):
167
+ logger.info("Soft-Linking %s to %s",
168
+ s.rh.container.localpath(), innc(number=i))
169
+ self.system.softlink(s.rh.container.localpath(), innc(number=i))
170
+
171
+ def prepare_namelist_delta(self, rh, namcontents, namlocal):
172
+ """Update the namelists with EDA related macros."""
173
+ nam_updated = super(IFSEdaAbstractAlgo, self).prepare_namelist_delta(rh,
174
+ namcontents,
175
+ namlocal)
176
+ if self.actual_nbe is not None:
177
+ self._set_nam_macro(namcontents, namlocal, 'NBE', self.actual_nbe)
178
+ nam_updated = True
179
+ return nam_updated
180
+
181
+ def prepare(self, rh, opts):
182
+ """Check the input files and act on it."""
183
+ self.system.subtitle('Solving the input files nightmare...')
184
+ if self.modelstate_needs_renumbering(rh):
185
+ eff_sections = self._members_effective_inputs()
186
+ if len(eff_sections) < self.actual_nbmin:
187
+ raise AlgoComponentError('Not enough input files to continue...')
188
+ logger.info("Starting input files renumbering. %d effective members found",
189
+ len(eff_sections))
190
+ self.modelstate_renumbering(rh, eff_sections)
191
+ self.system.subtitle('Other IFS related settings')
192
+ super().prepare(rh, opts)
193
+
194
+
195
+ class IFSEdaLaggedEnsembleAbstractAlgo(IFSEdaEnsembleAbstractAlgo):
196
+ """Base class for any EDA related task wrapped into an IFS/Arpege binary.
197
+
198
+ This extends the :class:`IFSEdaEnsembleAbstractAlgo` with a *nblag* attribute
199
+ and the ability to detect the input files and re-number them (in order to be
200
+ able to deal with missing members).
201
+ """
202
+
203
+ _PADDING_ROLE = 'PaddingModelState'
204
+
205
+ _abstract = True
206
+ _footprint = dict(
207
+ info='Base class for any EDA related task',
208
+ attr=dict(
209
+ nblag = dict(
210
+ info = "The number of lagged dates (auto-detected if omitted)",
211
+ type = int,
212
+ optional = True,
213
+ ),
214
+ padding = dict(
215
+ info = "Fill the gaps with some kind of climatological data",
216
+ type = bool,
217
+ optional = True,
218
+ default = False,
219
+ )
220
+ )
221
+ )
222
+
223
+ def __init__(self, *kargs, **kwargs):
224
+ super().__init__(*kargs, **kwargs)
225
+ self._actual_nresx = self.nblag
226
+
227
+ @property
228
+ def actual_nresx(self):
229
+ """The effective number of lagged dates."""
230
+ return self._actual_nresx
231
+
232
+ @property
233
+ def actual_totalnumber(self):
234
+ """The total number of members."""
235
+ return self.actual_nbe * self.actual_nresx if self.padding else self.actual_nbe
236
+
237
+ @property
238
+ def actual_nbmin(self):
239
+ """The minimum number of effective members that are mandatory to go one."""
240
+ return self.nbmin * self.actual_nresx if self.padding else self.nbmin
241
+
242
+ def _members_sorting_key(self, s):
243
+ """Return the sorting key for the **s** section."""
244
+ stuple = list(super()._members_sorting_key(s))
245
+ rdate = getattr(s.rh.resource, 'date')
246
+ stuple.insert(0, rdate)
247
+ return tuple(stuple)
248
+
249
+ def modelstate_needs_renumbering(self, rh):
250
+ """Check if, for the **mlist** members list, some renaming id needed."""
251
+ all_sections = self._members_all_inputs()
252
+ eff_sections = self._members_effective_inputs()
253
+
254
+ # Look for available dates (lagged ensemble)
255
+ all_dates = {sec.rh.resource.date for sec in all_sections}
256
+ if all_dates and self.nblag is not None:
257
+ # Consistency check
258
+ if len(all_dates) != self.nblag:
259
+ logger.warning('Discrepancy between *nblag* and input files...' +
260
+ ' sticking with *nblag*')
261
+ elif all_dates and self.nblag is None:
262
+ self._actual_nresx = len(all_dates)
263
+
264
+ # Fetch the effective members list
265
+ if self.padding:
266
+ # For each date, check the member's list
267
+ d_blocks = dict()
268
+ for a_date in all_dates:
269
+ d_sections = [sec for sec in all_sections if sec.rh.resource.date == a_date]
270
+ d_members = sorted([sec.rh.provider.member for sec in d_sections])
271
+ d_formats = {sec.rh.container.actualfmt for sec in d_sections}
272
+ d_blocks[a_date] = (d_members, d_formats)
273
+ ref_date, (eff_members, eff_formats) = d_blocks.popitem()
274
+ for a_date, a_data in d_blocks.items():
275
+ self.algoassert(a_data[0] == eff_members,
276
+ "Inconsistent members list (date={!s} vs {!s})."
277
+ .format(a_date, ref_date))
278
+ self.algoassert(a_data[1] == eff_formats,
279
+ "Inconsistent formats list (date={!s} vs {!s})."
280
+ .format(a_date, ref_date))
281
+ d_blocks[ref_date] = (eff_members, eff_formats)
282
+ if eff_members and self.nbmember is None:
283
+ # Here, NBE is the number of members for one date
284
+ self._actual_nbe = len(eff_members)
285
+ # Generate a complete list off input files
286
+ eff_members = []
287
+ for a_date, a_data in sorted(d_blocks.items()):
288
+ for member in a_data[0]:
289
+ eff_members.append((a_date, member))
290
+ else:
291
+ eff_members = [(sec.rh.resource.date, sec.rh.provider.member)
292
+ for sec in eff_sections]
293
+ eff_formats = {sec.rh.container.actualfmt for sec in eff_sections}
294
+ if eff_members and self.nbmember is None:
295
+ # Here, NBE is the number of members for all dates
296
+ self._actual_nbe = len(eff_members)
297
+
298
+ return self._check_members_list_numbering(rh, eff_members, eff_formats)
299
+
300
+ def modelstate_renumbering(self, rh, mlist):
301
+ """Actualy rename the effective inputs."""
302
+ if self.padding:
303
+ eff_format = mlist[0].rh.container.actualfmt
304
+ innc = self.naming_convention(kind='edainput', variant=self.kind, rh=rh,
305
+ totalnumber=self.actual_totalnumber,
306
+ actualfmt=eff_format)
307
+ all_sections = self._members_all_inputs()
308
+ paddingstuff = self.context.sequence.effective_inputs(role=self._PADDING_ROLE)
309
+ for i, s in enumerate(all_sections, start=1):
310
+ if s.stage == 'get' and s.rh.container.exists():
311
+ logger.info("Soft-Linking %s to %s", s.rh.container.localpath(),
312
+ innc(number=i))
313
+ self.system.softlink(s.rh.container.localpath(), innc(number=i))
314
+ else:
315
+ mypadding = None
316
+ for p in paddingstuff:
317
+ if getattr(p.rh.resource, 'ipert',
318
+ getattr(p.rh.resource, 'number', None)) == i:
319
+ mypadding = p
320
+ break
321
+ else:
322
+ if (getattr(p.rh.resource, 'date', None) == s.rh.resource.date and
323
+ getattr(p.rh.provider, 'member', None) == s.rh.provider.member):
324
+ mypadding = p
325
+ break
326
+ if mypadding is not None:
327
+ logger.warning("Soft-Linking Padding data %s to %s",
328
+ mypadding.rh.container.localpath(),
329
+ innc(number=i))
330
+ self.system.softlink(mypadding.rh.container.localpath(),
331
+ innc(number=i))
332
+ else:
333
+ raise AlgoComponentError('No padding data where found for i= {:d}: {!s}'
334
+ .format(i, s))
335
+ else:
336
+ super().modelstate_renumbering(rh, mlist)
337
+
338
+ def prepare_namelist_delta(self, rh, namcontents, namlocal):
339
+ """Update the namelists with EDA related macros."""
340
+ nam_updated = super().prepare_namelist_delta(rh, namcontents, namlocal)
341
+ if self.actual_nresx is not None:
342
+ self._set_nam_macro(namcontents, namlocal, 'NRESX', self.actual_nresx)
343
+ nam_updated = True
344
+ return nam_updated
345
+
346
+
347
+ class IFSEdaFemars(IFSEdaAbstractAlgo):
348
+ """Convert some FA file in ECMWF-GRIB files. PLEASE DO NOT USE !"""
349
+
350
+ _footprint = dict(
351
+ info='Convert some FA file in ECMWF-GRIB files.',
352
+ attr=dict(
353
+ kind=dict(
354
+ values=['femars'],
355
+ ),
356
+ rawfiles = dict(
357
+ type = bool,
358
+ optional = True,
359
+ default = False,
360
+ ),
361
+ )
362
+ )
363
+
364
+ def postfix(self, rh, opts):
365
+ """Find out if any special resources have been produced."""
366
+ sh = self.system
367
+ # Gather rawfiles in folders
368
+ if self.rawfiles:
369
+ flist = sh.glob('tmprawfile_D000_L*')
370
+ dest = 'rawfiles'
371
+ logger.info('Creating a rawfiles pack: %s', dest)
372
+ sh.mkdir(dest)
373
+ for fic in flist:
374
+ sh.mv(fic, dest, fmt='grib')
375
+ super().postfix(rh, opts)
376
+
377
+
378
+ class IFSInflationLike(IFSEdaAbstractAlgo):
379
+ """Apply the inflation scheme on a given modelstate."""
380
+
381
+ _RUNSTORE = 'RUNOUT'
382
+ _USELESS_MATCH = re.compile(r'^(?P<target>\w+)\+term\d+:\d+$')
383
+
384
+ _footprint = dict(
385
+ info='Operations around the background error covariance matrix',
386
+ attr=dict(
387
+ kind=dict(
388
+ values=['infl', 'pert', ],
389
+ ),
390
+ conf=dict(
391
+ values=[701, ]
392
+ )
393
+ )
394
+ )
395
+
396
+ def __init__(self, *kargs, **kwargs):
397
+ super().__init__(*kargs, **kwargs)
398
+ self._outputs_shelf = list()
399
+
400
+ def _check_effective_terms(self, roles):
401
+ eff_terms = None
402
+ for role in roles:
403
+ eterm = {sec.rh.resource.term
404
+ for sec in self.context.sequence.effective_inputs(role=role)}
405
+ if eterm:
406
+ if eff_terms is None:
407
+ eff_terms = eterm
408
+ else:
409
+ if eff_terms != eterm:
410
+ raise AlgoComponentError("Inconsistencies between inputs effective terms.")
411
+ return sorted(eff_terms)
412
+
413
+ def _link_stuff_in(self, role, actualterm, targetnc, targetintent='in', wastebasket=None):
414
+ estuff = [sec
415
+ for sec in self.context.sequence.effective_inputs(role=role)
416
+ if sec.rh.resource.term == actualterm]
417
+ if len(estuff) > 1:
418
+ logger.warning('Multiple %s for the same date ! Going on...', role)
419
+ elif len(estuff) == 1:
420
+ # Detect the inputs format
421
+ actfmt = estuff[0].rh.container.actualfmt
422
+ nconv = self.naming_convention(actualfmt=actfmt, **targetnc)
423
+ targetname = nconv(**targetnc)
424
+ if self.system.path.exists(targetname):
425
+ logger.info("%s: %s already exists. Hopping for the best...",
426
+ role, targetname)
427
+ else:
428
+ logger.info("%s: copying (intent=%s) %s to %s", role, targetintent,
429
+ estuff[0].rh.container.localpath(), targetname)
430
+ self.system.cp(estuff[0].rh.container.localpath(), targetname,
431
+ fmt=actfmt, intent=targetintent)
432
+ if wastebasket is not None:
433
+ wastebasket.append((targetname, actfmt))
434
+ return nconv, targetname, estuff[0]
435
+ return None, None, None
436
+
437
+ def execute(self, rh, opts):
438
+ """Loop on the various terms provided."""
439
+
440
+ eff_terms = self._check_effective_terms(['ModelState',
441
+ 'EnsembleMean',
442
+ 'Guess'])
443
+ fix_curclim = self.do_climfile_fixer(rh, convkind='modelclim')
444
+ fix_clclim = self.do_climfile_fixer(rh, convkind='closest_modelclim')
445
+
446
+ if eff_terms:
447
+ for actualterm in eff_terms:
448
+ wastebasket = list()
449
+ self.system.title('Loop on term {!s}'.format(actualterm))
450
+ self.system.subtitle('Solving the input files nightmare...')
451
+ # Ensemble Mean ?
452
+ mean_number = 2 if self.model == 'arome' else 0
453
+ targetnc = dict(kind='edainput', variant='infl', rh=rh, number=mean_number)
454
+ self._link_stuff_in('EnsembleMean', actualterm, targetnc,
455
+ wastebasket=wastebasket)
456
+ # Model State ?
457
+ targetnc = dict(kind='edainput', variant='infl', rh=rh, number=1)
458
+ _, _, mstate = self._link_stuff_in('ModelState', actualterm, targetnc,
459
+ wastebasket=wastebasket)
460
+ # Control ?
461
+ control_number = 0 if self.model == 'arome' else 2
462
+ targetnc = dict(kind='edainput', variant='infl', rh=rh, number=control_number)
463
+ self._link_stuff_in('Control', actualterm, targetnc,
464
+ wastebasket=wastebasket)
465
+ # Guess ?
466
+ targetnc = dict(kind='edaoutput', variant='infl', rh=rh, number=1, term=Time(0))
467
+ outnc, _, _ = self._link_stuff_in('Guess', actualterm, targetnc,
468
+ targetintent='inout')
469
+ if outnc is None:
470
+ outnc = self.naming_convention(kind='edaoutput', variant='infl', rh=rh)
471
+ # Fix clim !
472
+ if fix_curclim and mstate:
473
+ month = Month((mstate.rh.resource.date + actualterm).ymdh)
474
+ self.climfile_fixer(rh=rh, convkind='modelclim', month=month,
475
+ inputrole=('GlobalClim', 'InitialClim'),
476
+ inputkind='clim_model')
477
+ if fix_clclim and mstate:
478
+ closestmonth = Month((mstate.rh.resource.date + actualterm).ymdh + ':closest')
479
+ self.climfile_fixer(rh=rh, convkind='closest_modelclim', month=closestmonth,
480
+ inputrole=('GlobalClim', 'InitialClim'),
481
+ inputkind='clim_model')
482
+ # Deal with useless stuff... SADLY !
483
+ useless = [sec
484
+ for sec in self.context.sequence.effective_inputs(role='Useless')
485
+ if (sec.rh.resource.term == actualterm and
486
+ self._USELESS_MATCH.match(sec.rh.container.localpath()))]
487
+ for a_useless in useless:
488
+ targetname = self._USELESS_MATCH.match(a_useless.rh.container.localpath()).group('target')
489
+ if self.system.path.exists(targetname):
490
+ logger.warning("Some useless stuff is already here: %s. I don't care...",
491
+ targetname)
492
+ else:
493
+ logger.info("Dealing with useless stuff: %s -> %s",
494
+ a_useless.rh.container.localpath(), targetname)
495
+ self.system.cp(a_useless.rh.container.localpath(), targetname,
496
+ fmt=a_useless.rh.container.actualfmt, intent='in')
497
+ wastebasket.append((targetname, a_useless.rh.container.actualfmt))
498
+
499
+ # Standard execution
500
+ super().execute(rh, opts)
501
+
502
+ # The concatenated listing
503
+ self.system.cat('NODE.001_01', output='NODE.all')
504
+
505
+ # prepares the next execution
506
+ if len(eff_terms) > 1:
507
+ self.system.mkdir(self._RUNSTORE)
508
+ # Freeze the current output
509
+ shelf_label = self.system.path.join(self._RUNSTORE, outnc(number=1, term=actualterm))
510
+ self.system.move(outnc(number=1, term=Time(0)), shelf_label, fmt='fa')
511
+ self._outputs_shelf.append(shelf_label)
512
+ # Some cleaning
513
+ for afile in wastebasket:
514
+ self.system.remove(afile[0], fmt=afile[1])
515
+ self.system.rmall('ncf927', 'dirlst')
516
+ else:
517
+ # We should not be here but whatever... some task are poorly written !
518
+ super().execute(rh, opts)
519
+
520
+ def postfix(self, rh, opts):
521
+ """Post-processing cleaning."""
522
+ self.system.title('Finalising the execution...')
523
+ for afile in self._outputs_shelf:
524
+ logger.info("Output found: %s", self.system.path.basename(afile))
525
+ self.system.move(afile, self.system.path.basename(afile), fmt='fa')
526
+ super().postfix(rh, opts)
527
+
528
+
529
+ class IFSInflationFactor(IFSEdaEnsembleAbstractAlgo):
530
+ """Compute an inflation factor based on individual members."""
531
+
532
+ _footprint = dict(
533
+ info='Compute an inflation factor based on individual members',
534
+ attr=dict(
535
+ kind=dict(
536
+ values=['infl_factor', ],
537
+ ),
538
+ )
539
+ )
540
+
541
+
542
+ class IFSInflationFactorLegacy(IFSInflationFactor):
543
+ """Compute an inflation factor based on individual members. KEPT FOR COMPATIBILITY.
544
+
545
+ DO NOT USE !
546
+ """
547
+
548
+ _footprint = dict(
549
+ info='Compute an inflation factor based on individual members',
550
+ attr=dict(
551
+ kind=dict(
552
+ values=['infl', 'pert'],
553
+ ),
554
+ conf=dict(
555
+ outcast=[701, ]
556
+ )
557
+ )
558
+ )
559
+
560
+
561
+ class IFSEnsembleMean(IFSEdaEnsembleAbstractAlgo):
562
+ """Apply the inflation scheme on a given modelstate."""
563
+
564
+ _footprint = dict(
565
+ info='Operations around the background error covariance matrix',
566
+ attr=dict(
567
+ kind=dict(
568
+ values=['mean', ],
569
+ ),
570
+ )
571
+ )
572
+
573
+
574
+ class IFSCovB(IFSEdaLaggedEnsembleAbstractAlgo):
575
+ """Operations around the background error covariance matrix."""
576
+
577
+ _footprint = dict(
578
+ info='Operations around the background error covariance matrix',
579
+ attr=dict(
580
+ kind=dict(
581
+ values=['covb', ],
582
+ ),
583
+ hybrid = dict(
584
+ type = bool,
585
+ optional = True,
586
+ default = False,
587
+ ),
588
+ )
589
+ )
590
+
591
+ _HYBRID_CLIM_ROLE = 'ClimatologicalModelState'
592
+
593
+ @property
594
+ def actual_totalnumber(self):
595
+ """The total number of members (times 2 if hybrid...)."""
596
+ parent_totalnumber = super().actual_totalnumber
597
+ return parent_totalnumber * 2 if self.hybrid else parent_totalnumber
598
+
599
+ def prepare(self, rh, opts):
600
+ """Default pre-link for the initial condition file"""
601
+ super().prepare(rh, opts)
602
+ # Legacy...
603
+ for num, sec in enumerate(sorted(self.context.sequence.effective_inputs(role='Rawfiles'),
604
+ key=self._members_sorting_key),
605
+ start=1):
606
+ repname = sec.rh.container.localpath()
607
+ radical = repname.split('_')[0] + '_D{:03d}_L{:s}'
608
+ for filename in self.system.listdir(repname):
609
+ level = re.search(r'_L(\d+)$', filename)
610
+ if level is not None:
611
+ self.system.softlink(self.system.path.join(repname, filename),
612
+ radical.format(num, level.group(1)))
613
+ # Legacy...
614
+ for num, sec in enumerate(sorted(self.context.sequence.effective_inputs(role='LaggedEnsemble'),
615
+ key=self._members_sorting_key),
616
+ start=1):
617
+ repname = sec.rh.container.localpath()
618
+ radical = repname.split('_')[0] + '_{:03d}'
619
+ self.system.softlink(repname, radical.format(num))
620
+ # Requesting Hybrid computations ?
621
+ if self.hybrid:
622
+ hybstuff = self.context.sequence.effective_inputs(role=self._HYBRID_CLIM_ROLE)
623
+ hybformat = hybstuff[0].rh.container.actualfmt
624
+ totalnumber = self.actual_nbe * self.actual_nresx if self.padding else self.actual_nbe
625
+ for i, tnum in enumerate(range(totalnumber + 1, 2 * totalnumber + 1)):
626
+ innc = self.naming_convention(kind='edainput', variant=self.kind,
627
+ totalnumber=self.actual_totalnumber,
628
+ rh=rh, actualfmt=hybformat)
629
+ logger.info("Soft-Linking %s to %s",
630
+ hybstuff[i].rh.container.localpath(), innc(number=tnum))
631
+ self.system.softlink(hybstuff[i].rh.container.localpath(),
632
+ innc(number=tnum))