vortex-nwp 2.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (144) hide show
  1. vortex/__init__.py +159 -0
  2. vortex/algo/__init__.py +13 -0
  3. vortex/algo/components.py +2462 -0
  4. vortex/algo/mpitools.py +1953 -0
  5. vortex/algo/mpitools_templates/__init__.py +1 -0
  6. vortex/algo/mpitools_templates/envelope_wrapper_default.tpl +27 -0
  7. vortex/algo/mpitools_templates/envelope_wrapper_mpiauto.tpl +29 -0
  8. vortex/algo/mpitools_templates/wrapstd_wrapper_default.tpl +18 -0
  9. vortex/algo/serversynctools.py +171 -0
  10. vortex/config.py +112 -0
  11. vortex/data/__init__.py +19 -0
  12. vortex/data/abstractstores.py +1510 -0
  13. vortex/data/containers.py +835 -0
  14. vortex/data/contents.py +622 -0
  15. vortex/data/executables.py +275 -0
  16. vortex/data/flow.py +119 -0
  17. vortex/data/geometries.ini +2689 -0
  18. vortex/data/geometries.py +799 -0
  19. vortex/data/handlers.py +1230 -0
  20. vortex/data/outflow.py +67 -0
  21. vortex/data/providers.py +487 -0
  22. vortex/data/resources.py +207 -0
  23. vortex/data/stores.py +1390 -0
  24. vortex/data/sync_templates/__init__.py +0 -0
  25. vortex/gloves.py +309 -0
  26. vortex/layout/__init__.py +20 -0
  27. vortex/layout/contexts.py +577 -0
  28. vortex/layout/dataflow.py +1220 -0
  29. vortex/layout/monitor.py +969 -0
  30. vortex/nwp/__init__.py +14 -0
  31. vortex/nwp/algo/__init__.py +21 -0
  32. vortex/nwp/algo/assim.py +537 -0
  33. vortex/nwp/algo/clim.py +1086 -0
  34. vortex/nwp/algo/coupling.py +831 -0
  35. vortex/nwp/algo/eda.py +840 -0
  36. vortex/nwp/algo/eps.py +785 -0
  37. vortex/nwp/algo/forecasts.py +886 -0
  38. vortex/nwp/algo/fpserver.py +1303 -0
  39. vortex/nwp/algo/ifsnaming.py +463 -0
  40. vortex/nwp/algo/ifsroot.py +404 -0
  41. vortex/nwp/algo/monitoring.py +263 -0
  42. vortex/nwp/algo/mpitools.py +694 -0
  43. vortex/nwp/algo/odbtools.py +1258 -0
  44. vortex/nwp/algo/oopsroot.py +916 -0
  45. vortex/nwp/algo/oopstests.py +220 -0
  46. vortex/nwp/algo/request.py +660 -0
  47. vortex/nwp/algo/stdpost.py +1641 -0
  48. vortex/nwp/data/__init__.py +30 -0
  49. vortex/nwp/data/assim.py +380 -0
  50. vortex/nwp/data/boundaries.py +314 -0
  51. vortex/nwp/data/climfiles.py +521 -0
  52. vortex/nwp/data/configfiles.py +153 -0
  53. vortex/nwp/data/consts.py +954 -0
  54. vortex/nwp/data/ctpini.py +149 -0
  55. vortex/nwp/data/diagnostics.py +209 -0
  56. vortex/nwp/data/eda.py +147 -0
  57. vortex/nwp/data/eps.py +432 -0
  58. vortex/nwp/data/executables.py +1045 -0
  59. vortex/nwp/data/fields.py +111 -0
  60. vortex/nwp/data/gridfiles.py +380 -0
  61. vortex/nwp/data/logs.py +584 -0
  62. vortex/nwp/data/modelstates.py +363 -0
  63. vortex/nwp/data/monitoring.py +193 -0
  64. vortex/nwp/data/namelists.py +696 -0
  65. vortex/nwp/data/obs.py +840 -0
  66. vortex/nwp/data/oopsexec.py +74 -0
  67. vortex/nwp/data/providers.py +207 -0
  68. vortex/nwp/data/query.py +206 -0
  69. vortex/nwp/data/stores.py +160 -0
  70. vortex/nwp/data/surfex.py +337 -0
  71. vortex/nwp/syntax/__init__.py +9 -0
  72. vortex/nwp/syntax/stdattrs.py +437 -0
  73. vortex/nwp/tools/__init__.py +10 -0
  74. vortex/nwp/tools/addons.py +40 -0
  75. vortex/nwp/tools/agt.py +67 -0
  76. vortex/nwp/tools/bdap.py +59 -0
  77. vortex/nwp/tools/bdcp.py +41 -0
  78. vortex/nwp/tools/bdm.py +24 -0
  79. vortex/nwp/tools/bdmp.py +54 -0
  80. vortex/nwp/tools/conftools.py +1661 -0
  81. vortex/nwp/tools/drhook.py +66 -0
  82. vortex/nwp/tools/grib.py +294 -0
  83. vortex/nwp/tools/gribdiff.py +104 -0
  84. vortex/nwp/tools/ifstools.py +203 -0
  85. vortex/nwp/tools/igastuff.py +273 -0
  86. vortex/nwp/tools/mars.py +68 -0
  87. vortex/nwp/tools/odb.py +657 -0
  88. vortex/nwp/tools/partitioning.py +258 -0
  89. vortex/nwp/tools/satrad.py +71 -0
  90. vortex/nwp/util/__init__.py +6 -0
  91. vortex/nwp/util/async.py +212 -0
  92. vortex/nwp/util/beacon.py +40 -0
  93. vortex/nwp/util/diffpygram.py +447 -0
  94. vortex/nwp/util/ens.py +279 -0
  95. vortex/nwp/util/hooks.py +139 -0
  96. vortex/nwp/util/taskdeco.py +85 -0
  97. vortex/nwp/util/usepygram.py +697 -0
  98. vortex/nwp/util/usetnt.py +101 -0
  99. vortex/proxy.py +6 -0
  100. vortex/sessions.py +374 -0
  101. vortex/syntax/__init__.py +9 -0
  102. vortex/syntax/stdattrs.py +867 -0
  103. vortex/syntax/stddeco.py +185 -0
  104. vortex/toolbox.py +1117 -0
  105. vortex/tools/__init__.py +20 -0
  106. vortex/tools/actions.py +523 -0
  107. vortex/tools/addons.py +316 -0
  108. vortex/tools/arm.py +96 -0
  109. vortex/tools/compression.py +325 -0
  110. vortex/tools/date.py +27 -0
  111. vortex/tools/ddhpack.py +10 -0
  112. vortex/tools/delayedactions.py +782 -0
  113. vortex/tools/env.py +541 -0
  114. vortex/tools/folder.py +834 -0
  115. vortex/tools/grib.py +738 -0
  116. vortex/tools/lfi.py +953 -0
  117. vortex/tools/listings.py +423 -0
  118. vortex/tools/names.py +637 -0
  119. vortex/tools/net.py +2124 -0
  120. vortex/tools/odb.py +10 -0
  121. vortex/tools/parallelism.py +368 -0
  122. vortex/tools/prestaging.py +210 -0
  123. vortex/tools/rawfiles.py +10 -0
  124. vortex/tools/schedulers.py +480 -0
  125. vortex/tools/services.py +940 -0
  126. vortex/tools/storage.py +996 -0
  127. vortex/tools/surfex.py +61 -0
  128. vortex/tools/systems.py +3976 -0
  129. vortex/tools/targets.py +440 -0
  130. vortex/util/__init__.py +9 -0
  131. vortex/util/config.py +1122 -0
  132. vortex/util/empty.py +24 -0
  133. vortex/util/helpers.py +216 -0
  134. vortex/util/introspection.py +69 -0
  135. vortex/util/iosponge.py +80 -0
  136. vortex/util/roles.py +49 -0
  137. vortex/util/storefunctions.py +129 -0
  138. vortex/util/structs.py +26 -0
  139. vortex/util/worker.py +162 -0
  140. vortex_nwp-2.0.0.dist-info/METADATA +67 -0
  141. vortex_nwp-2.0.0.dist-info/RECORD +144 -0
  142. vortex_nwp-2.0.0.dist-info/WHEEL +5 -0
  143. vortex_nwp-2.0.0.dist-info/licenses/LICENSE +517 -0
  144. vortex_nwp-2.0.0.dist-info/top_level.txt +1 -0
vortex/nwp/algo/eda.py ADDED
@@ -0,0 +1,840 @@
1
+ """
2
+ AlgoComponents dedicated to computations related to the Ensemble Data Assimilation
3
+ system.
4
+ """
5
+
6
+ import math
7
+ import re
8
+
9
+ from bronx.fancies import loggers
10
+ from bronx.stdtypes.date import Month, Time
11
+ import footprints
12
+
13
+ from vortex.algo.components import AlgoComponentError
14
+ from .ifsroot import IFSParallel
15
+
16
+ #: Automatic export off
17
+ __all__ = []
18
+
19
+ logger = loggers.getLogger(__name__)
20
+
21
+
22
+ class IFSEdaAbstractAlgo(IFSParallel):
23
+ """Base class for any EDA related task wrapped into an IFS/Arpege binary."""
24
+
25
+ _abstract = True
26
+ _footprint = dict(
27
+ info="Base class for any EDA related task",
28
+ attr=dict(
29
+ inputnaming=dict(
30
+ info="Prescribe your own naming template for input files.",
31
+ optional=True,
32
+ doc_visibility=footprints.doc.visibility.ADVANCED,
33
+ ),
34
+ ),
35
+ )
36
+
37
+ def naming_convention(self, kind, rh, actualfmt=None, **kwargs):
38
+ """Take into account the *inputnaming* attribute."""
39
+ if kind == "edainput":
40
+ return super().naming_convention(
41
+ kind,
42
+ rh,
43
+ actualfmt=actualfmt,
44
+ namingformat=self.inputnaming,
45
+ **kwargs,
46
+ )
47
+ else:
48
+ return super().naming_convention(
49
+ kind, rh, actualfmt=actualfmt, **kwargs
50
+ )
51
+
52
+
53
+ class IFSEdaEnsembleAbstractAlgo(IFSEdaAbstractAlgo):
54
+ """Base class for any EDA related task wrapped into an IFS/Arpege binary.
55
+
56
+ This extends the :class:`IFSEdaAbstractAlgo` with a *nbmember* attribute and
57
+ the ability to detect the input files and re-number them (in order to be able
58
+ to deal with missing members).
59
+ """
60
+
61
+ _INPUTS_ROLE = "ModelState"
62
+
63
+ _abstract = True
64
+ _footprint = dict(
65
+ info="Base class for any EDA related task",
66
+ attr=dict(
67
+ nbmember=dict(
68
+ info="The number of members to deal will (auto-detected if omitted)",
69
+ type=int,
70
+ optional=True,
71
+ ),
72
+ nbmin=dict(
73
+ info="The minimum number of input members that is mandatory to go one",
74
+ type=int,
75
+ optional=True,
76
+ default=2,
77
+ ),
78
+ ),
79
+ )
80
+
81
+ def __init__(self, *kargs, **kwargs):
82
+ super().__init__(*kargs, **kwargs)
83
+ self._actual_nbe = self.nbmember
84
+
85
+ @property
86
+ def actual_nbe(self):
87
+ """The effective number of members."""
88
+ return self._actual_nbe
89
+
90
+ @property
91
+ def actual_totalnumber(self):
92
+ """The total number of members (=! actual_nbe for lagged ensembles. see below)."""
93
+ return self.actual_nbe
94
+
95
+ @property
96
+ def actual_nbmin(self):
97
+ """The minimum number of effective members that are mandatory to go one."""
98
+ return self.nbmin
99
+
100
+ @staticmethod
101
+ def _members_sorting_key(s):
102
+ """Return the sorting key for the **s** section."""
103
+ member = getattr(s.rh.provider, "member", None)
104
+ member = -math.inf if member is None else member
105
+ block = getattr(s.rh.provider, "block", "")
106
+ filename = getattr(s.rh.container, "basename", "")
107
+ return member, block, filename
108
+
109
+ def _members_effective_inputs(self):
110
+ """The list of effective sections representing input data."""
111
+ return sorted(
112
+ self.context.sequence.effective_inputs(role=self._INPUTS_ROLE),
113
+ key=self._members_sorting_key,
114
+ )
115
+
116
+ def _members_all_inputs(self):
117
+ """The list of sections representing input data."""
118
+ return sorted(
119
+ self.context.sequence.filtered_inputs(role=self._INPUTS_ROLE),
120
+ key=self._members_sorting_key,
121
+ )
122
+
123
+ def _check_members_list_numbering(self, rh, mlist, mformats):
124
+ """Check if, for the **mlist** members list, some renaming id needed."""
125
+ if len(set(mlist)) != len(mlist):
126
+ logger.warning(
127
+ "Some members are duplicated. That's very strange..."
128
+ )
129
+ if self.nbmember is None:
130
+ self.algoassert(
131
+ len(mformats) <= 1, "Mixed formats are not allowed !"
132
+ )
133
+ elif self.nbmember and len(mformats) > 1:
134
+ logger.info(
135
+ "%s have mixed formats... please correct that.",
136
+ self._INPUTS_ROLE,
137
+ )
138
+ if mlist and self.nbmember is not None:
139
+ # Consistency check
140
+ if len(mlist) != self.nbmember:
141
+ logger.warning(
142
+ "Discrepancy between *nbmember* and effective input files..."
143
+ + " sticking with *nbmember*"
144
+ )
145
+ else:
146
+ logger.info("The input files member numbers checks out !")
147
+ return False # Ok, apparently the user knows what she/he is doing
148
+ elif self.nbmember and not mlist:
149
+ return False # Ok, apparently the user knows what she/he is doing
150
+ elif mlist and self.nbmember is None:
151
+ innc = self.naming_convention(
152
+ kind="edainput",
153
+ variant=self.kind,
154
+ rh=rh,
155
+ totalnumber=self.actual_totalnumber,
156
+ actualfmt=mformats.pop(),
157
+ )
158
+ checkfiles = [
159
+ m
160
+ for m in range(1, len(mlist) + 1)
161
+ if self.system.path.exists(innc(number=m))
162
+ ]
163
+ if len(checkfiles) == len(mlist):
164
+ logger.info("The input files numbering checks out !")
165
+ return (
166
+ False # Ok, apparently the user knows what she/he is doing
167
+ )
168
+ elif len(checkfiles) == 0:
169
+ return True
170
+ else:
171
+ raise AlgoComponentError(
172
+ "Members renumbering is needed but some "
173
+ + "files are blocking the way !"
174
+ )
175
+ elif len(mlist) == 0 and self.nbmember is None:
176
+ raise AlgoComponentError("No input files where found !")
177
+
178
+ def modelstate_needs_renumbering(self, rh):
179
+ """Check if, for the **mlist** members list, some renaming id needed."""
180
+ eff_sections = self._members_effective_inputs()
181
+ eff_members = [sec.rh.provider.member for sec in eff_sections]
182
+ eff_formats = {sec.rh.container.actualfmt for sec in eff_sections}
183
+ if eff_members and self.nbmember is None:
184
+ self._actual_nbe = len(eff_members)
185
+ return self._check_members_list_numbering(rh, eff_members, eff_formats)
186
+
187
+ def modelstate_renumbering(self, rh, mlist):
188
+ """Actualy rename the effective inputs."""
189
+ eff_format = mlist[0].rh.container.actualfmt
190
+ innc = self.naming_convention(
191
+ kind="edainput",
192
+ variant=self.kind,
193
+ rh=rh,
194
+ totalnumber=self.actual_totalnumber,
195
+ actualfmt=eff_format,
196
+ )
197
+ for i, s in enumerate(mlist, start=1):
198
+ logger.info(
199
+ "Soft-Linking %s to %s",
200
+ s.rh.container.localpath(),
201
+ innc(number=i),
202
+ )
203
+ self.system.softlink(s.rh.container.localpath(), innc(number=i))
204
+
205
+ def prepare_namelist_delta(self, rh, namcontents, namlocal):
206
+ """Update the namelists with EDA related macros."""
207
+ nam_updated = super(IFSEdaAbstractAlgo, self).prepare_namelist_delta(
208
+ rh, namcontents, namlocal
209
+ )
210
+ if self.actual_nbe is not None:
211
+ self._set_nam_macro(namcontents, namlocal, "NBE", self.actual_nbe)
212
+ nam_updated = True
213
+ return nam_updated
214
+
215
+ def prepare(self, rh, opts):
216
+ """Check the input files and act on it."""
217
+ self.system.subtitle("Solving the input files nightmare...")
218
+ if self.modelstate_needs_renumbering(rh):
219
+ eff_sections = self._members_effective_inputs()
220
+ if len(eff_sections) < self.actual_nbmin:
221
+ raise AlgoComponentError(
222
+ "Not enough input files to continue..."
223
+ )
224
+ logger.info(
225
+ "Starting input files renumbering. %d effective members found",
226
+ len(eff_sections),
227
+ )
228
+ self.modelstate_renumbering(rh, eff_sections)
229
+ self.system.subtitle("Other IFS related settings")
230
+ super().prepare(rh, opts)
231
+
232
+
233
+ class IFSEdaLaggedEnsembleAbstractAlgo(IFSEdaEnsembleAbstractAlgo):
234
+ """Base class for any EDA related task wrapped into an IFS/Arpege binary.
235
+
236
+ This extends the :class:`IFSEdaEnsembleAbstractAlgo` with a *nblag* attribute
237
+ and the ability to detect the input files and re-number them (in order to be
238
+ able to deal with missing members).
239
+ """
240
+
241
+ _PADDING_ROLE = "PaddingModelState"
242
+
243
+ _abstract = True
244
+ _footprint = dict(
245
+ info="Base class for any EDA related task",
246
+ attr=dict(
247
+ nblag=dict(
248
+ info="The number of lagged dates (auto-detected if omitted)",
249
+ type=int,
250
+ optional=True,
251
+ ),
252
+ padding=dict(
253
+ info="Fill the gaps with some kind of climatological data",
254
+ type=bool,
255
+ optional=True,
256
+ default=False,
257
+ ),
258
+ ),
259
+ )
260
+
261
+ def __init__(self, *kargs, **kwargs):
262
+ super().__init__(*kargs, **kwargs)
263
+ self._actual_nresx = self.nblag
264
+
265
+ @property
266
+ def actual_nresx(self):
267
+ """The effective number of lagged dates."""
268
+ return self._actual_nresx
269
+
270
+ @property
271
+ def actual_totalnumber(self):
272
+ """The total number of members."""
273
+ return (
274
+ self.actual_nbe * self.actual_nresx
275
+ if self.padding
276
+ else self.actual_nbe
277
+ )
278
+
279
+ @property
280
+ def actual_nbmin(self):
281
+ """The minimum number of effective members that are mandatory to go one."""
282
+ return self.nbmin * self.actual_nresx if self.padding else self.nbmin
283
+
284
+ def _members_sorting_key(self, s):
285
+ """Return the sorting key for the **s** section."""
286
+ stuple = list(super()._members_sorting_key(s))
287
+ rdate = getattr(s.rh.resource, "date")
288
+ stuple.insert(0, rdate)
289
+ return tuple(stuple)
290
+
291
+ def modelstate_needs_renumbering(self, rh):
292
+ """Check if, for the **mlist** members list, some renaming id needed."""
293
+ all_sections = self._members_all_inputs()
294
+ eff_sections = self._members_effective_inputs()
295
+
296
+ # Look for available dates (lagged ensemble)
297
+ all_dates = {sec.rh.resource.date for sec in all_sections}
298
+ if all_dates and self.nblag is not None:
299
+ # Consistency check
300
+ if len(all_dates) != self.nblag:
301
+ logger.warning(
302
+ "Discrepancy between *nblag* and input files..."
303
+ + " sticking with *nblag*"
304
+ )
305
+ elif all_dates and self.nblag is None:
306
+ self._actual_nresx = len(all_dates)
307
+
308
+ # Fetch the effective members list
309
+ if self.padding:
310
+ # For each date, check the member's list
311
+ d_blocks = dict()
312
+ for a_date in all_dates:
313
+ d_sections = [
314
+ sec
315
+ for sec in all_sections
316
+ if sec.rh.resource.date == a_date
317
+ ]
318
+ d_members = sorted(
319
+ [sec.rh.provider.member for sec in d_sections]
320
+ )
321
+ d_formats = {sec.rh.container.actualfmt for sec in d_sections}
322
+ d_blocks[a_date] = (d_members, d_formats)
323
+ ref_date, (eff_members, eff_formats) = d_blocks.popitem()
324
+ for a_date, a_data in d_blocks.items():
325
+ self.algoassert(
326
+ a_data[0] == eff_members,
327
+ "Inconsistent members list (date={!s} vs {!s}).".format(
328
+ a_date, ref_date
329
+ ),
330
+ )
331
+ self.algoassert(
332
+ a_data[1] == eff_formats,
333
+ "Inconsistent formats list (date={!s} vs {!s}).".format(
334
+ a_date, ref_date
335
+ ),
336
+ )
337
+ d_blocks[ref_date] = (eff_members, eff_formats)
338
+ if eff_members and self.nbmember is None:
339
+ # Here, NBE is the number of members for one date
340
+ self._actual_nbe = len(eff_members)
341
+ # Generate a complete list off input files
342
+ eff_members = []
343
+ for a_date, a_data in sorted(d_blocks.items()):
344
+ for member in a_data[0]:
345
+ eff_members.append((a_date, member))
346
+ else:
347
+ eff_members = [
348
+ (sec.rh.resource.date, sec.rh.provider.member)
349
+ for sec in eff_sections
350
+ ]
351
+ eff_formats = {sec.rh.container.actualfmt for sec in eff_sections}
352
+ if eff_members and self.nbmember is None:
353
+ # Here, NBE is the number of members for all dates
354
+ self._actual_nbe = len(eff_members)
355
+
356
+ return self._check_members_list_numbering(rh, eff_members, eff_formats)
357
+
358
+ def modelstate_renumbering(self, rh, mlist):
359
+ """Actualy rename the effective inputs."""
360
+ if self.padding:
361
+ eff_format = mlist[0].rh.container.actualfmt
362
+ innc = self.naming_convention(
363
+ kind="edainput",
364
+ variant=self.kind,
365
+ rh=rh,
366
+ totalnumber=self.actual_totalnumber,
367
+ actualfmt=eff_format,
368
+ )
369
+ all_sections = self._members_all_inputs()
370
+ paddingstuff = self.context.sequence.effective_inputs(
371
+ role=self._PADDING_ROLE
372
+ )
373
+ for i, s in enumerate(all_sections, start=1):
374
+ if s.stage == "get" and s.rh.container.exists():
375
+ logger.info(
376
+ "Soft-Linking %s to %s",
377
+ s.rh.container.localpath(),
378
+ innc(number=i),
379
+ )
380
+ self.system.softlink(
381
+ s.rh.container.localpath(), innc(number=i)
382
+ )
383
+ else:
384
+ mypadding = None
385
+ for p in paddingstuff:
386
+ if (
387
+ getattr(
388
+ p.rh.resource,
389
+ "ipert",
390
+ getattr(p.rh.resource, "number", None),
391
+ )
392
+ == i
393
+ ):
394
+ mypadding = p
395
+ break
396
+ else:
397
+ if (
398
+ getattr(p.rh.resource, "date", None)
399
+ == s.rh.resource.date
400
+ and getattr(p.rh.provider, "member", None)
401
+ == s.rh.provider.member
402
+ ):
403
+ mypadding = p
404
+ break
405
+ if mypadding is not None:
406
+ logger.warning(
407
+ "Soft-Linking Padding data %s to %s",
408
+ mypadding.rh.container.localpath(),
409
+ innc(number=i),
410
+ )
411
+ self.system.softlink(
412
+ mypadding.rh.container.localpath(), innc(number=i)
413
+ )
414
+ else:
415
+ raise AlgoComponentError(
416
+ "No padding data where found for i= {:d}: {!s}".format(
417
+ i, s
418
+ )
419
+ )
420
+ else:
421
+ super().modelstate_renumbering(rh, mlist)
422
+
423
+ def prepare_namelist_delta(self, rh, namcontents, namlocal):
424
+ """Update the namelists with EDA related macros."""
425
+ nam_updated = super().prepare_namelist_delta(rh, namcontents, namlocal)
426
+ if self.actual_nresx is not None:
427
+ self._set_nam_macro(
428
+ namcontents, namlocal, "NRESX", self.actual_nresx
429
+ )
430
+ nam_updated = True
431
+ return nam_updated
432
+
433
+
434
+ class IFSEdaFemars(IFSEdaAbstractAlgo):
435
+ """Convert some FA file in ECMWF-GRIB files. PLEASE DO NOT USE !"""
436
+
437
+ _footprint = dict(
438
+ info="Convert some FA file in ECMWF-GRIB files.",
439
+ attr=dict(
440
+ kind=dict(
441
+ values=["femars"],
442
+ ),
443
+ rawfiles=dict(
444
+ type=bool,
445
+ optional=True,
446
+ default=False,
447
+ ),
448
+ ),
449
+ )
450
+
451
+ def postfix(self, rh, opts):
452
+ """Find out if any special resources have been produced."""
453
+ sh = self.system
454
+ # Gather rawfiles in folders
455
+ if self.rawfiles:
456
+ flist = sh.glob("tmprawfile_D000_L*")
457
+ dest = "rawfiles"
458
+ logger.info("Creating a rawfiles pack: %s", dest)
459
+ sh.mkdir(dest)
460
+ for fic in flist:
461
+ sh.mv(fic, dest, fmt="grib")
462
+ super().postfix(rh, opts)
463
+
464
+
465
+ class IFSInflationLike(IFSEdaAbstractAlgo):
466
+ """Apply the inflation scheme on a given modelstate."""
467
+
468
+ _RUNSTORE = "RUNOUT"
469
+ _USELESS_MATCH = re.compile(r"^(?P<target>\w+)\+term\d+:\d+$")
470
+
471
+ _footprint = dict(
472
+ info="Operations around the background error covariance matrix",
473
+ attr=dict(
474
+ kind=dict(
475
+ values=[
476
+ "infl",
477
+ "pert",
478
+ ],
479
+ ),
480
+ conf=dict(
481
+ values=[
482
+ 701,
483
+ ]
484
+ ),
485
+ ),
486
+ )
487
+
488
+ def __init__(self, *kargs, **kwargs):
489
+ super().__init__(*kargs, **kwargs)
490
+ self._outputs_shelf = list()
491
+
492
+ def _check_effective_terms(self, roles):
493
+ eff_terms = None
494
+ for role in roles:
495
+ eterm = {
496
+ sec.rh.resource.term
497
+ for sec in self.context.sequence.effective_inputs(role=role)
498
+ }
499
+ if eterm:
500
+ if eff_terms is None:
501
+ eff_terms = eterm
502
+ else:
503
+ if eff_terms != eterm:
504
+ raise AlgoComponentError(
505
+ "Inconsistencies between inputs effective terms."
506
+ )
507
+ return sorted(eff_terms)
508
+
509
+ def _link_stuff_in(
510
+ self, role, actualterm, targetnc, targetintent="in", wastebasket=None
511
+ ):
512
+ estuff = [
513
+ sec
514
+ for sec in self.context.sequence.effective_inputs(role=role)
515
+ if sec.rh.resource.term == actualterm
516
+ ]
517
+ if len(estuff) > 1:
518
+ logger.warning(
519
+ "Multiple %s for the same date ! Going on...", role
520
+ )
521
+ elif len(estuff) == 1:
522
+ # Detect the inputs format
523
+ actfmt = estuff[0].rh.container.actualfmt
524
+ nconv = self.naming_convention(actualfmt=actfmt, **targetnc)
525
+ targetname = nconv(**targetnc)
526
+ if self.system.path.exists(targetname):
527
+ logger.info(
528
+ "%s: %s already exists. Hopping for the best...",
529
+ role,
530
+ targetname,
531
+ )
532
+ else:
533
+ logger.info(
534
+ "%s: copying (intent=%s) %s to %s",
535
+ role,
536
+ targetintent,
537
+ estuff[0].rh.container.localpath(),
538
+ targetname,
539
+ )
540
+ self.system.cp(
541
+ estuff[0].rh.container.localpath(),
542
+ targetname,
543
+ fmt=actfmt,
544
+ intent=targetintent,
545
+ )
546
+ if wastebasket is not None:
547
+ wastebasket.append((targetname, actfmt))
548
+ return nconv, targetname, estuff[0]
549
+ return None, None, None
550
+
551
+ def execute(self, rh, opts):
552
+ """Loop on the various terms provided."""
553
+
554
+ eff_terms = self._check_effective_terms(
555
+ ["ModelState", "EnsembleMean", "Guess"]
556
+ )
557
+ fix_curclim = self.do_climfile_fixer(rh, convkind="modelclim")
558
+ fix_clclim = self.do_climfile_fixer(rh, convkind="closest_modelclim")
559
+
560
+ if eff_terms:
561
+ for actualterm in eff_terms:
562
+ wastebasket = list()
563
+ self.system.title("Loop on term {!s}".format(actualterm))
564
+ self.system.subtitle("Solving the input files nightmare...")
565
+ # Ensemble Mean ?
566
+ mean_number = 2 if self.model == "arome" else 0
567
+ targetnc = dict(
568
+ kind="edainput", variant="infl", rh=rh, number=mean_number
569
+ )
570
+ self._link_stuff_in(
571
+ "EnsembleMean",
572
+ actualterm,
573
+ targetnc,
574
+ wastebasket=wastebasket,
575
+ )
576
+ # Model State ?
577
+ targetnc = dict(
578
+ kind="edainput", variant="infl", rh=rh, number=1
579
+ )
580
+ _, _, mstate = self._link_stuff_in(
581
+ "ModelState", actualterm, targetnc, wastebasket=wastebasket
582
+ )
583
+ # Control ?
584
+ control_number = 0 if self.model == "arome" else 2
585
+ targetnc = dict(
586
+ kind="edainput",
587
+ variant="infl",
588
+ rh=rh,
589
+ number=control_number,
590
+ )
591
+ self._link_stuff_in(
592
+ "Control", actualterm, targetnc, wastebasket=wastebasket
593
+ )
594
+ # Guess ?
595
+ targetnc = dict(
596
+ kind="edaoutput",
597
+ variant="infl",
598
+ rh=rh,
599
+ number=1,
600
+ term=Time(0),
601
+ )
602
+ outnc, _, _ = self._link_stuff_in(
603
+ "Guess", actualterm, targetnc, targetintent="inout"
604
+ )
605
+ if outnc is None:
606
+ outnc = self.naming_convention(
607
+ kind="edaoutput", variant="infl", rh=rh
608
+ )
609
+ # Fix clim !
610
+ if fix_curclim and mstate:
611
+ month = Month((mstate.rh.resource.date + actualterm).ymdh)
612
+ self.climfile_fixer(
613
+ rh=rh,
614
+ convkind="modelclim",
615
+ month=month,
616
+ inputrole=("GlobalClim", "InitialClim"),
617
+ inputkind="clim_model",
618
+ )
619
+ if fix_clclim and mstate:
620
+ closestmonth = Month(
621
+ (mstate.rh.resource.date + actualterm).ymdh
622
+ + ":closest"
623
+ )
624
+ self.climfile_fixer(
625
+ rh=rh,
626
+ convkind="closest_modelclim",
627
+ month=closestmonth,
628
+ inputrole=("GlobalClim", "InitialClim"),
629
+ inputkind="clim_model",
630
+ )
631
+ # Deal with useless stuff... SADLY !
632
+ useless = [
633
+ sec
634
+ for sec in self.context.sequence.effective_inputs(
635
+ role="Useless"
636
+ )
637
+ if (
638
+ sec.rh.resource.term == actualterm
639
+ and self._USELESS_MATCH.match(
640
+ sec.rh.container.localpath()
641
+ )
642
+ )
643
+ ]
644
+ for a_useless in useless:
645
+ targetname = self._USELESS_MATCH.match(
646
+ a_useless.rh.container.localpath()
647
+ ).group("target")
648
+ if self.system.path.exists(targetname):
649
+ logger.warning(
650
+ "Some useless stuff is already here: %s. I don't care...",
651
+ targetname,
652
+ )
653
+ else:
654
+ logger.info(
655
+ "Dealing with useless stuff: %s -> %s",
656
+ a_useless.rh.container.localpath(),
657
+ targetname,
658
+ )
659
+ self.system.cp(
660
+ a_useless.rh.container.localpath(),
661
+ targetname,
662
+ fmt=a_useless.rh.container.actualfmt,
663
+ intent="in",
664
+ )
665
+ wastebasket.append(
666
+ (targetname, a_useless.rh.container.actualfmt)
667
+ )
668
+
669
+ # Standard execution
670
+ super().execute(rh, opts)
671
+
672
+ # The concatenated listing
673
+ self.system.cat("NODE.001_01", output="NODE.all")
674
+
675
+ # prepares the next execution
676
+ if len(eff_terms) > 1:
677
+ self.system.mkdir(self._RUNSTORE)
678
+ # Freeze the current output
679
+ shelf_label = self.system.path.join(
680
+ self._RUNSTORE, outnc(number=1, term=actualterm)
681
+ )
682
+ self.system.move(
683
+ outnc(number=1, term=Time(0)), shelf_label, fmt="fa"
684
+ )
685
+ self._outputs_shelf.append(shelf_label)
686
+ # Some cleaning
687
+ for afile in wastebasket:
688
+ self.system.remove(afile[0], fmt=afile[1])
689
+ self.system.rmall("ncf927", "dirlst")
690
+ else:
691
+ # We should not be here but whatever... some task are poorly written !
692
+ super().execute(rh, opts)
693
+
694
+ def postfix(self, rh, opts):
695
+ """Post-processing cleaning."""
696
+ self.system.title("Finalising the execution...")
697
+ for afile in self._outputs_shelf:
698
+ logger.info("Output found: %s", self.system.path.basename(afile))
699
+ self.system.move(afile, self.system.path.basename(afile), fmt="fa")
700
+ super().postfix(rh, opts)
701
+
702
+
703
+ class IFSInflationFactor(IFSEdaEnsembleAbstractAlgo):
704
+ """Compute an inflation factor based on individual members."""
705
+
706
+ _footprint = dict(
707
+ info="Compute an inflation factor based on individual members",
708
+ attr=dict(
709
+ kind=dict(
710
+ values=[
711
+ "infl_factor",
712
+ ],
713
+ ),
714
+ ),
715
+ )
716
+
717
+
718
+ class IFSInflationFactorLegacy(IFSInflationFactor):
719
+ """Compute an inflation factor based on individual members. KEPT FOR COMPATIBILITY.
720
+
721
+ DO NOT USE !
722
+ """
723
+
724
+ _footprint = dict(
725
+ info="Compute an inflation factor based on individual members",
726
+ attr=dict(
727
+ kind=dict(
728
+ values=["infl", "pert"],
729
+ ),
730
+ conf=dict(
731
+ outcast=[
732
+ 701,
733
+ ]
734
+ ),
735
+ ),
736
+ )
737
+
738
+
739
+ class IFSEnsembleMean(IFSEdaEnsembleAbstractAlgo):
740
+ """Apply the inflation scheme on a given modelstate."""
741
+
742
+ _footprint = dict(
743
+ info="Operations around the background error covariance matrix",
744
+ attr=dict(
745
+ kind=dict(
746
+ values=[
747
+ "mean",
748
+ ],
749
+ ),
750
+ ),
751
+ )
752
+
753
+
754
+ class IFSCovB(IFSEdaLaggedEnsembleAbstractAlgo):
755
+ """Operations around the background error covariance matrix."""
756
+
757
+ _footprint = dict(
758
+ info="Operations around the background error covariance matrix",
759
+ attr=dict(
760
+ kind=dict(
761
+ values=[
762
+ "covb",
763
+ ],
764
+ ),
765
+ hybrid=dict(
766
+ type=bool,
767
+ optional=True,
768
+ default=False,
769
+ ),
770
+ ),
771
+ )
772
+
773
+ _HYBRID_CLIM_ROLE = "ClimatologicalModelState"
774
+
775
+ @property
776
+ def actual_totalnumber(self):
777
+ """The total number of members (times 2 if hybrid...)."""
778
+ parent_totalnumber = super().actual_totalnumber
779
+ return parent_totalnumber * 2 if self.hybrid else parent_totalnumber
780
+
781
+ def prepare(self, rh, opts):
782
+ """Default pre-link for the initial condition file"""
783
+ super().prepare(rh, opts)
784
+ # Legacy...
785
+ for num, sec in enumerate(
786
+ sorted(
787
+ self.context.sequence.effective_inputs(role="Rawfiles"),
788
+ key=self._members_sorting_key,
789
+ ),
790
+ start=1,
791
+ ):
792
+ repname = sec.rh.container.localpath()
793
+ radical = repname.split("_")[0] + "_D{:03d}_L{:s}"
794
+ for filename in self.system.listdir(repname):
795
+ level = re.search(r"_L(\d+)$", filename)
796
+ if level is not None:
797
+ self.system.softlink(
798
+ self.system.path.join(repname, filename),
799
+ radical.format(num, level.group(1)),
800
+ )
801
+ # Legacy...
802
+ for num, sec in enumerate(
803
+ sorted(
804
+ self.context.sequence.effective_inputs(role="LaggedEnsemble"),
805
+ key=self._members_sorting_key,
806
+ ),
807
+ start=1,
808
+ ):
809
+ repname = sec.rh.container.localpath()
810
+ radical = repname.split("_")[0] + "_{:03d}"
811
+ self.system.softlink(repname, radical.format(num))
812
+ # Requesting Hybrid computations ?
813
+ if self.hybrid:
814
+ hybstuff = self.context.sequence.effective_inputs(
815
+ role=self._HYBRID_CLIM_ROLE
816
+ )
817
+ hybformat = hybstuff[0].rh.container.actualfmt
818
+ totalnumber = (
819
+ self.actual_nbe * self.actual_nresx
820
+ if self.padding
821
+ else self.actual_nbe
822
+ )
823
+ for i, tnum in enumerate(
824
+ range(totalnumber + 1, 2 * totalnumber + 1)
825
+ ):
826
+ innc = self.naming_convention(
827
+ kind="edainput",
828
+ variant=self.kind,
829
+ totalnumber=self.actual_totalnumber,
830
+ rh=rh,
831
+ actualfmt=hybformat,
832
+ )
833
+ logger.info(
834
+ "Soft-Linking %s to %s",
835
+ hybstuff[i].rh.container.localpath(),
836
+ innc(number=tnum),
837
+ )
838
+ self.system.softlink(
839
+ hybstuff[i].rh.container.localpath(), innc(number=tnum)
840
+ )