vortex-nwp 2.0.0b1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- vortex/__init__.py +135 -0
- vortex/algo/__init__.py +12 -0
- vortex/algo/components.py +2136 -0
- vortex/algo/mpitools.py +1648 -0
- vortex/algo/mpitools_templates/envelope_wrapper_default.tpl +27 -0
- vortex/algo/mpitools_templates/envelope_wrapper_mpiauto.tpl +29 -0
- vortex/algo/mpitools_templates/wrapstd_wrapper_default.tpl +18 -0
- vortex/algo/serversynctools.py +170 -0
- vortex/config.py +115 -0
- vortex/data/__init__.py +13 -0
- vortex/data/abstractstores.py +1572 -0
- vortex/data/containers.py +780 -0
- vortex/data/contents.py +596 -0
- vortex/data/executables.py +284 -0
- vortex/data/flow.py +113 -0
- vortex/data/geometries.ini +2689 -0
- vortex/data/geometries.py +703 -0
- vortex/data/handlers.py +1021 -0
- vortex/data/outflow.py +67 -0
- vortex/data/providers.py +465 -0
- vortex/data/resources.py +201 -0
- vortex/data/stores.py +1271 -0
- vortex/gloves.py +282 -0
- vortex/layout/__init__.py +27 -0
- vortex/layout/appconf.py +109 -0
- vortex/layout/contexts.py +511 -0
- vortex/layout/dataflow.py +1069 -0
- vortex/layout/jobs.py +1276 -0
- vortex/layout/monitor.py +833 -0
- vortex/layout/nodes.py +1424 -0
- vortex/layout/subjobs.py +464 -0
- vortex/nwp/__init__.py +11 -0
- vortex/nwp/algo/__init__.py +12 -0
- vortex/nwp/algo/assim.py +483 -0
- vortex/nwp/algo/clim.py +920 -0
- vortex/nwp/algo/coupling.py +609 -0
- vortex/nwp/algo/eda.py +632 -0
- vortex/nwp/algo/eps.py +613 -0
- vortex/nwp/algo/forecasts.py +745 -0
- vortex/nwp/algo/fpserver.py +927 -0
- vortex/nwp/algo/ifsnaming.py +403 -0
- vortex/nwp/algo/ifsroot.py +311 -0
- vortex/nwp/algo/monitoring.py +202 -0
- vortex/nwp/algo/mpitools.py +554 -0
- vortex/nwp/algo/odbtools.py +974 -0
- vortex/nwp/algo/oopsroot.py +735 -0
- vortex/nwp/algo/oopstests.py +186 -0
- vortex/nwp/algo/request.py +579 -0
- vortex/nwp/algo/stdpost.py +1285 -0
- vortex/nwp/data/__init__.py +12 -0
- vortex/nwp/data/assim.py +392 -0
- vortex/nwp/data/boundaries.py +261 -0
- vortex/nwp/data/climfiles.py +539 -0
- vortex/nwp/data/configfiles.py +149 -0
- vortex/nwp/data/consts.py +929 -0
- vortex/nwp/data/ctpini.py +133 -0
- vortex/nwp/data/diagnostics.py +181 -0
- vortex/nwp/data/eda.py +148 -0
- vortex/nwp/data/eps.py +383 -0
- vortex/nwp/data/executables.py +1039 -0
- vortex/nwp/data/fields.py +96 -0
- vortex/nwp/data/gridfiles.py +308 -0
- vortex/nwp/data/logs.py +551 -0
- vortex/nwp/data/modelstates.py +334 -0
- vortex/nwp/data/monitoring.py +220 -0
- vortex/nwp/data/namelists.py +644 -0
- vortex/nwp/data/obs.py +748 -0
- vortex/nwp/data/oopsexec.py +72 -0
- vortex/nwp/data/providers.py +182 -0
- vortex/nwp/data/query.py +217 -0
- vortex/nwp/data/stores.py +147 -0
- vortex/nwp/data/surfex.py +338 -0
- vortex/nwp/syntax/__init__.py +9 -0
- vortex/nwp/syntax/stdattrs.py +375 -0
- vortex/nwp/tools/__init__.py +10 -0
- vortex/nwp/tools/addons.py +35 -0
- vortex/nwp/tools/agt.py +55 -0
- vortex/nwp/tools/bdap.py +48 -0
- vortex/nwp/tools/bdcp.py +38 -0
- vortex/nwp/tools/bdm.py +21 -0
- vortex/nwp/tools/bdmp.py +49 -0
- vortex/nwp/tools/conftools.py +1311 -0
- vortex/nwp/tools/drhook.py +62 -0
- vortex/nwp/tools/grib.py +268 -0
- vortex/nwp/tools/gribdiff.py +99 -0
- vortex/nwp/tools/ifstools.py +163 -0
- vortex/nwp/tools/igastuff.py +249 -0
- vortex/nwp/tools/mars.py +56 -0
- vortex/nwp/tools/odb.py +548 -0
- vortex/nwp/tools/partitioning.py +234 -0
- vortex/nwp/tools/satrad.py +56 -0
- vortex/nwp/util/__init__.py +6 -0
- vortex/nwp/util/async.py +184 -0
- vortex/nwp/util/beacon.py +40 -0
- vortex/nwp/util/diffpygram.py +359 -0
- vortex/nwp/util/ens.py +198 -0
- vortex/nwp/util/hooks.py +128 -0
- vortex/nwp/util/taskdeco.py +81 -0
- vortex/nwp/util/usepygram.py +591 -0
- vortex/nwp/util/usetnt.py +87 -0
- vortex/proxy.py +6 -0
- vortex/sessions.py +341 -0
- vortex/syntax/__init__.py +9 -0
- vortex/syntax/stdattrs.py +628 -0
- vortex/syntax/stddeco.py +176 -0
- vortex/toolbox.py +982 -0
- vortex/tools/__init__.py +11 -0
- vortex/tools/actions.py +457 -0
- vortex/tools/addons.py +297 -0
- vortex/tools/arm.py +76 -0
- vortex/tools/compression.py +322 -0
- vortex/tools/date.py +20 -0
- vortex/tools/ddhpack.py +10 -0
- vortex/tools/delayedactions.py +672 -0
- vortex/tools/env.py +513 -0
- vortex/tools/folder.py +663 -0
- vortex/tools/grib.py +559 -0
- vortex/tools/lfi.py +746 -0
- vortex/tools/listings.py +354 -0
- vortex/tools/names.py +575 -0
- vortex/tools/net.py +1790 -0
- vortex/tools/odb.py +10 -0
- vortex/tools/parallelism.py +336 -0
- vortex/tools/prestaging.py +186 -0
- vortex/tools/rawfiles.py +10 -0
- vortex/tools/schedulers.py +413 -0
- vortex/tools/services.py +871 -0
- vortex/tools/storage.py +1061 -0
- vortex/tools/surfex.py +61 -0
- vortex/tools/systems.py +3396 -0
- vortex/tools/targets.py +384 -0
- vortex/util/__init__.py +9 -0
- vortex/util/config.py +1071 -0
- vortex/util/empty.py +24 -0
- vortex/util/helpers.py +184 -0
- vortex/util/introspection.py +63 -0
- vortex/util/iosponge.py +76 -0
- vortex/util/roles.py +51 -0
- vortex/util/storefunctions.py +103 -0
- vortex/util/structs.py +26 -0
- vortex/util/worker.py +150 -0
- vortex_nwp-2.0.0b1.dist-info/LICENSE +517 -0
- vortex_nwp-2.0.0b1.dist-info/METADATA +50 -0
- vortex_nwp-2.0.0b1.dist-info/RECORD +146 -0
- vortex_nwp-2.0.0b1.dist-info/WHEEL +5 -0
- vortex_nwp-2.0.0b1.dist-info/top_level.txt +1 -0
vortex/nwp/algo/eda.py
ADDED
|
@@ -0,0 +1,632 @@
|
|
|
1
|
+
"""
|
|
2
|
+
AlgoComponents dedicated to computations related to the Ensemble Data Assimilation
|
|
3
|
+
system.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
import math
|
|
7
|
+
import re
|
|
8
|
+
|
|
9
|
+
from bronx.fancies import loggers
|
|
10
|
+
from bronx.stdtypes.date import Month, Time
|
|
11
|
+
import footprints
|
|
12
|
+
|
|
13
|
+
from vortex.algo.components import AlgoComponentError
|
|
14
|
+
from .ifsroot import IFSParallel
|
|
15
|
+
|
|
16
|
+
#: Automatic export off
|
|
17
|
+
__all__ = []
|
|
18
|
+
|
|
19
|
+
logger = loggers.getLogger(__name__)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class IFSEdaAbstractAlgo(IFSParallel):
|
|
23
|
+
"""Base class for any EDA related task wrapped into an IFS/Arpege binary."""
|
|
24
|
+
|
|
25
|
+
_abstract = True
|
|
26
|
+
_footprint = dict(
|
|
27
|
+
info='Base class for any EDA related task',
|
|
28
|
+
attr=dict(
|
|
29
|
+
inputnaming = dict(
|
|
30
|
+
info = 'Prescribe your own naming template for input files.',
|
|
31
|
+
optional = True,
|
|
32
|
+
doc_visibility = footprints.doc.visibility.ADVANCED,
|
|
33
|
+
),
|
|
34
|
+
)
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
def naming_convention(self, kind, rh, actualfmt=None, **kwargs):
|
|
38
|
+
"""Take into account the *inputnaming* attribute."""
|
|
39
|
+
if kind == 'edainput':
|
|
40
|
+
return super().naming_convention(kind, rh,
|
|
41
|
+
actualfmt=actualfmt,
|
|
42
|
+
namingformat=self.inputnaming,
|
|
43
|
+
**kwargs)
|
|
44
|
+
else:
|
|
45
|
+
return super().naming_convention(kind, rh,
|
|
46
|
+
actualfmt=actualfmt,
|
|
47
|
+
**kwargs)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class IFSEdaEnsembleAbstractAlgo(IFSEdaAbstractAlgo):
|
|
51
|
+
"""Base class for any EDA related task wrapped into an IFS/Arpege binary.
|
|
52
|
+
|
|
53
|
+
This extends the :class:`IFSEdaAbstractAlgo` with a *nbmember* attribute and
|
|
54
|
+
the ability to detect the input files and re-number them (in order to be able
|
|
55
|
+
to deal with missing members).
|
|
56
|
+
"""
|
|
57
|
+
|
|
58
|
+
_INPUTS_ROLE = 'ModelState'
|
|
59
|
+
|
|
60
|
+
_abstract = True
|
|
61
|
+
_footprint = dict(
|
|
62
|
+
info='Base class for any EDA related task',
|
|
63
|
+
attr=dict(
|
|
64
|
+
nbmember = dict(
|
|
65
|
+
info = "The number of members to deal will (auto-detected if omitted)",
|
|
66
|
+
type = int,
|
|
67
|
+
optional = True,
|
|
68
|
+
),
|
|
69
|
+
nbmin = dict(
|
|
70
|
+
info = "The minimum number of input members that is mandatory to go one",
|
|
71
|
+
type = int,
|
|
72
|
+
optional = True,
|
|
73
|
+
default = 2,
|
|
74
|
+
)
|
|
75
|
+
)
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
def __init__(self, *kargs, **kwargs):
|
|
79
|
+
super().__init__(*kargs, **kwargs)
|
|
80
|
+
self._actual_nbe = self.nbmember
|
|
81
|
+
|
|
82
|
+
@property
|
|
83
|
+
def actual_nbe(self):
|
|
84
|
+
"""The effective number of members."""
|
|
85
|
+
return self._actual_nbe
|
|
86
|
+
|
|
87
|
+
@property
|
|
88
|
+
def actual_totalnumber(self):
|
|
89
|
+
"""The total number of members (=! actual_nbe for lagged ensembles. see below)."""
|
|
90
|
+
return self.actual_nbe
|
|
91
|
+
|
|
92
|
+
@property
|
|
93
|
+
def actual_nbmin(self):
|
|
94
|
+
"""The minimum number of effective members that are mandatory to go one."""
|
|
95
|
+
return self.nbmin
|
|
96
|
+
|
|
97
|
+
@staticmethod
|
|
98
|
+
def _members_sorting_key(s):
|
|
99
|
+
"""Return the sorting key for the **s** section."""
|
|
100
|
+
member = getattr(s.rh.provider, 'member', None)
|
|
101
|
+
member = - math.inf if member is None else member
|
|
102
|
+
block = getattr(s.rh.provider, 'block', '')
|
|
103
|
+
filename = getattr(s.rh.container, 'basename', '')
|
|
104
|
+
return member, block, filename
|
|
105
|
+
|
|
106
|
+
def _members_effective_inputs(self):
|
|
107
|
+
"""The list of effective sections representing input data."""
|
|
108
|
+
return sorted(self.context.sequence.effective_inputs(role=self._INPUTS_ROLE),
|
|
109
|
+
key=self._members_sorting_key)
|
|
110
|
+
|
|
111
|
+
def _members_all_inputs(self):
|
|
112
|
+
"""The list of sections representing input data."""
|
|
113
|
+
return sorted(self.context.sequence.filtered_inputs(role=self._INPUTS_ROLE),
|
|
114
|
+
key=self._members_sorting_key)
|
|
115
|
+
|
|
116
|
+
def _check_members_list_numbering(self, rh, mlist, mformats):
|
|
117
|
+
"""Check if, for the **mlist** members list, some renaming id needed."""
|
|
118
|
+
if len(set(mlist)) != len(mlist):
|
|
119
|
+
logger.warning("Some members are duplicated. That's very strange...")
|
|
120
|
+
if self.nbmember is None:
|
|
121
|
+
self.algoassert(len(mformats) <= 1, 'Mixed formats are not allowed !')
|
|
122
|
+
elif self.nbmember and len(mformats) > 1:
|
|
123
|
+
logger.info('%s have mixed formats... please correct that.', self._INPUTS_ROLE)
|
|
124
|
+
if mlist and self.nbmember is not None:
|
|
125
|
+
# Consistency check
|
|
126
|
+
if len(mlist) != self.nbmember:
|
|
127
|
+
logger.warning('Discrepancy between *nbmember* and effective input files...' +
|
|
128
|
+
' sticking with *nbmember*')
|
|
129
|
+
else:
|
|
130
|
+
logger.info("The input files member numbers checks out !")
|
|
131
|
+
return False # Ok, apparently the user knows what she/he is doing
|
|
132
|
+
elif self.nbmember and not mlist:
|
|
133
|
+
return False # Ok, apparently the user knows what she/he is doing
|
|
134
|
+
elif mlist and self.nbmember is None:
|
|
135
|
+
innc = self.naming_convention(kind='edainput', variant=self.kind, rh=rh,
|
|
136
|
+
totalnumber=self.actual_totalnumber,
|
|
137
|
+
actualfmt=mformats.pop())
|
|
138
|
+
checkfiles = [m for m in range(1, len(mlist) + 1)
|
|
139
|
+
if self.system.path.exists(innc(number=m))]
|
|
140
|
+
if len(checkfiles) == len(mlist):
|
|
141
|
+
logger.info("The input files numbering checks out !")
|
|
142
|
+
return False # Ok, apparently the user knows what she/he is doing
|
|
143
|
+
elif len(checkfiles) == 0:
|
|
144
|
+
return True
|
|
145
|
+
else:
|
|
146
|
+
raise AlgoComponentError('Members renumbering is needed but some ' +
|
|
147
|
+
'files are blocking the way !')
|
|
148
|
+
elif len(mlist) == 0 and self.nbmember is None:
|
|
149
|
+
raise AlgoComponentError('No input files where found !')
|
|
150
|
+
|
|
151
|
+
def modelstate_needs_renumbering(self, rh):
|
|
152
|
+
"""Check if, for the **mlist** members list, some renaming id needed."""
|
|
153
|
+
eff_sections = self._members_effective_inputs()
|
|
154
|
+
eff_members = [sec.rh.provider.member for sec in eff_sections]
|
|
155
|
+
eff_formats = {sec.rh.container.actualfmt for sec in eff_sections}
|
|
156
|
+
if eff_members and self.nbmember is None:
|
|
157
|
+
self._actual_nbe = len(eff_members)
|
|
158
|
+
return self._check_members_list_numbering(rh, eff_members, eff_formats)
|
|
159
|
+
|
|
160
|
+
def modelstate_renumbering(self, rh, mlist):
|
|
161
|
+
"""Actualy rename the effective inputs."""
|
|
162
|
+
eff_format = mlist[0].rh.container.actualfmt
|
|
163
|
+
innc = self.naming_convention(kind='edainput', variant=self.kind, rh=rh,
|
|
164
|
+
totalnumber=self.actual_totalnumber,
|
|
165
|
+
actualfmt=eff_format)
|
|
166
|
+
for i, s in enumerate(mlist, start=1):
|
|
167
|
+
logger.info("Soft-Linking %s to %s",
|
|
168
|
+
s.rh.container.localpath(), innc(number=i))
|
|
169
|
+
self.system.softlink(s.rh.container.localpath(), innc(number=i))
|
|
170
|
+
|
|
171
|
+
def prepare_namelist_delta(self, rh, namcontents, namlocal):
|
|
172
|
+
"""Update the namelists with EDA related macros."""
|
|
173
|
+
nam_updated = super(IFSEdaAbstractAlgo, self).prepare_namelist_delta(rh,
|
|
174
|
+
namcontents,
|
|
175
|
+
namlocal)
|
|
176
|
+
if self.actual_nbe is not None:
|
|
177
|
+
self._set_nam_macro(namcontents, namlocal, 'NBE', self.actual_nbe)
|
|
178
|
+
nam_updated = True
|
|
179
|
+
return nam_updated
|
|
180
|
+
|
|
181
|
+
def prepare(self, rh, opts):
|
|
182
|
+
"""Check the input files and act on it."""
|
|
183
|
+
self.system.subtitle('Solving the input files nightmare...')
|
|
184
|
+
if self.modelstate_needs_renumbering(rh):
|
|
185
|
+
eff_sections = self._members_effective_inputs()
|
|
186
|
+
if len(eff_sections) < self.actual_nbmin:
|
|
187
|
+
raise AlgoComponentError('Not enough input files to continue...')
|
|
188
|
+
logger.info("Starting input files renumbering. %d effective members found",
|
|
189
|
+
len(eff_sections))
|
|
190
|
+
self.modelstate_renumbering(rh, eff_sections)
|
|
191
|
+
self.system.subtitle('Other IFS related settings')
|
|
192
|
+
super().prepare(rh, opts)
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
class IFSEdaLaggedEnsembleAbstractAlgo(IFSEdaEnsembleAbstractAlgo):
|
|
196
|
+
"""Base class for any EDA related task wrapped into an IFS/Arpege binary.
|
|
197
|
+
|
|
198
|
+
This extends the :class:`IFSEdaEnsembleAbstractAlgo` with a *nblag* attribute
|
|
199
|
+
and the ability to detect the input files and re-number them (in order to be
|
|
200
|
+
able to deal with missing members).
|
|
201
|
+
"""
|
|
202
|
+
|
|
203
|
+
_PADDING_ROLE = 'PaddingModelState'
|
|
204
|
+
|
|
205
|
+
_abstract = True
|
|
206
|
+
_footprint = dict(
|
|
207
|
+
info='Base class for any EDA related task',
|
|
208
|
+
attr=dict(
|
|
209
|
+
nblag = dict(
|
|
210
|
+
info = "The number of lagged dates (auto-detected if omitted)",
|
|
211
|
+
type = int,
|
|
212
|
+
optional = True,
|
|
213
|
+
),
|
|
214
|
+
padding = dict(
|
|
215
|
+
info = "Fill the gaps with some kind of climatological data",
|
|
216
|
+
type = bool,
|
|
217
|
+
optional = True,
|
|
218
|
+
default = False,
|
|
219
|
+
)
|
|
220
|
+
)
|
|
221
|
+
)
|
|
222
|
+
|
|
223
|
+
def __init__(self, *kargs, **kwargs):
|
|
224
|
+
super().__init__(*kargs, **kwargs)
|
|
225
|
+
self._actual_nresx = self.nblag
|
|
226
|
+
|
|
227
|
+
@property
|
|
228
|
+
def actual_nresx(self):
|
|
229
|
+
"""The effective number of lagged dates."""
|
|
230
|
+
return self._actual_nresx
|
|
231
|
+
|
|
232
|
+
@property
|
|
233
|
+
def actual_totalnumber(self):
|
|
234
|
+
"""The total number of members."""
|
|
235
|
+
return self.actual_nbe * self.actual_nresx if self.padding else self.actual_nbe
|
|
236
|
+
|
|
237
|
+
@property
|
|
238
|
+
def actual_nbmin(self):
|
|
239
|
+
"""The minimum number of effective members that are mandatory to go one."""
|
|
240
|
+
return self.nbmin * self.actual_nresx if self.padding else self.nbmin
|
|
241
|
+
|
|
242
|
+
def _members_sorting_key(self, s):
|
|
243
|
+
"""Return the sorting key for the **s** section."""
|
|
244
|
+
stuple = list(super()._members_sorting_key(s))
|
|
245
|
+
rdate = getattr(s.rh.resource, 'date')
|
|
246
|
+
stuple.insert(0, rdate)
|
|
247
|
+
return tuple(stuple)
|
|
248
|
+
|
|
249
|
+
def modelstate_needs_renumbering(self, rh):
|
|
250
|
+
"""Check if, for the **mlist** members list, some renaming id needed."""
|
|
251
|
+
all_sections = self._members_all_inputs()
|
|
252
|
+
eff_sections = self._members_effective_inputs()
|
|
253
|
+
|
|
254
|
+
# Look for available dates (lagged ensemble)
|
|
255
|
+
all_dates = {sec.rh.resource.date for sec in all_sections}
|
|
256
|
+
if all_dates and self.nblag is not None:
|
|
257
|
+
# Consistency check
|
|
258
|
+
if len(all_dates) != self.nblag:
|
|
259
|
+
logger.warning('Discrepancy between *nblag* and input files...' +
|
|
260
|
+
' sticking with *nblag*')
|
|
261
|
+
elif all_dates and self.nblag is None:
|
|
262
|
+
self._actual_nresx = len(all_dates)
|
|
263
|
+
|
|
264
|
+
# Fetch the effective members list
|
|
265
|
+
if self.padding:
|
|
266
|
+
# For each date, check the member's list
|
|
267
|
+
d_blocks = dict()
|
|
268
|
+
for a_date in all_dates:
|
|
269
|
+
d_sections = [sec for sec in all_sections if sec.rh.resource.date == a_date]
|
|
270
|
+
d_members = sorted([sec.rh.provider.member for sec in d_sections])
|
|
271
|
+
d_formats = {sec.rh.container.actualfmt for sec in d_sections}
|
|
272
|
+
d_blocks[a_date] = (d_members, d_formats)
|
|
273
|
+
ref_date, (eff_members, eff_formats) = d_blocks.popitem()
|
|
274
|
+
for a_date, a_data in d_blocks.items():
|
|
275
|
+
self.algoassert(a_data[0] == eff_members,
|
|
276
|
+
"Inconsistent members list (date={!s} vs {!s})."
|
|
277
|
+
.format(a_date, ref_date))
|
|
278
|
+
self.algoassert(a_data[1] == eff_formats,
|
|
279
|
+
"Inconsistent formats list (date={!s} vs {!s})."
|
|
280
|
+
.format(a_date, ref_date))
|
|
281
|
+
d_blocks[ref_date] = (eff_members, eff_formats)
|
|
282
|
+
if eff_members and self.nbmember is None:
|
|
283
|
+
# Here, NBE is the number of members for one date
|
|
284
|
+
self._actual_nbe = len(eff_members)
|
|
285
|
+
# Generate a complete list off input files
|
|
286
|
+
eff_members = []
|
|
287
|
+
for a_date, a_data in sorted(d_blocks.items()):
|
|
288
|
+
for member in a_data[0]:
|
|
289
|
+
eff_members.append((a_date, member))
|
|
290
|
+
else:
|
|
291
|
+
eff_members = [(sec.rh.resource.date, sec.rh.provider.member)
|
|
292
|
+
for sec in eff_sections]
|
|
293
|
+
eff_formats = {sec.rh.container.actualfmt for sec in eff_sections}
|
|
294
|
+
if eff_members and self.nbmember is None:
|
|
295
|
+
# Here, NBE is the number of members for all dates
|
|
296
|
+
self._actual_nbe = len(eff_members)
|
|
297
|
+
|
|
298
|
+
return self._check_members_list_numbering(rh, eff_members, eff_formats)
|
|
299
|
+
|
|
300
|
+
def modelstate_renumbering(self, rh, mlist):
|
|
301
|
+
"""Actualy rename the effective inputs."""
|
|
302
|
+
if self.padding:
|
|
303
|
+
eff_format = mlist[0].rh.container.actualfmt
|
|
304
|
+
innc = self.naming_convention(kind='edainput', variant=self.kind, rh=rh,
|
|
305
|
+
totalnumber=self.actual_totalnumber,
|
|
306
|
+
actualfmt=eff_format)
|
|
307
|
+
all_sections = self._members_all_inputs()
|
|
308
|
+
paddingstuff = self.context.sequence.effective_inputs(role=self._PADDING_ROLE)
|
|
309
|
+
for i, s in enumerate(all_sections, start=1):
|
|
310
|
+
if s.stage == 'get' and s.rh.container.exists():
|
|
311
|
+
logger.info("Soft-Linking %s to %s", s.rh.container.localpath(),
|
|
312
|
+
innc(number=i))
|
|
313
|
+
self.system.softlink(s.rh.container.localpath(), innc(number=i))
|
|
314
|
+
else:
|
|
315
|
+
mypadding = None
|
|
316
|
+
for p in paddingstuff:
|
|
317
|
+
if getattr(p.rh.resource, 'ipert',
|
|
318
|
+
getattr(p.rh.resource, 'number', None)) == i:
|
|
319
|
+
mypadding = p
|
|
320
|
+
break
|
|
321
|
+
else:
|
|
322
|
+
if (getattr(p.rh.resource, 'date', None) == s.rh.resource.date and
|
|
323
|
+
getattr(p.rh.provider, 'member', None) == s.rh.provider.member):
|
|
324
|
+
mypadding = p
|
|
325
|
+
break
|
|
326
|
+
if mypadding is not None:
|
|
327
|
+
logger.warning("Soft-Linking Padding data %s to %s",
|
|
328
|
+
mypadding.rh.container.localpath(),
|
|
329
|
+
innc(number=i))
|
|
330
|
+
self.system.softlink(mypadding.rh.container.localpath(),
|
|
331
|
+
innc(number=i))
|
|
332
|
+
else:
|
|
333
|
+
raise AlgoComponentError('No padding data where found for i= {:d}: {!s}'
|
|
334
|
+
.format(i, s))
|
|
335
|
+
else:
|
|
336
|
+
super().modelstate_renumbering(rh, mlist)
|
|
337
|
+
|
|
338
|
+
def prepare_namelist_delta(self, rh, namcontents, namlocal):
|
|
339
|
+
"""Update the namelists with EDA related macros."""
|
|
340
|
+
nam_updated = super().prepare_namelist_delta(rh, namcontents, namlocal)
|
|
341
|
+
if self.actual_nresx is not None:
|
|
342
|
+
self._set_nam_macro(namcontents, namlocal, 'NRESX', self.actual_nresx)
|
|
343
|
+
nam_updated = True
|
|
344
|
+
return nam_updated
|
|
345
|
+
|
|
346
|
+
|
|
347
|
+
class IFSEdaFemars(IFSEdaAbstractAlgo):
|
|
348
|
+
"""Convert some FA file in ECMWF-GRIB files. PLEASE DO NOT USE !"""
|
|
349
|
+
|
|
350
|
+
_footprint = dict(
|
|
351
|
+
info='Convert some FA file in ECMWF-GRIB files.',
|
|
352
|
+
attr=dict(
|
|
353
|
+
kind=dict(
|
|
354
|
+
values=['femars'],
|
|
355
|
+
),
|
|
356
|
+
rawfiles = dict(
|
|
357
|
+
type = bool,
|
|
358
|
+
optional = True,
|
|
359
|
+
default = False,
|
|
360
|
+
),
|
|
361
|
+
)
|
|
362
|
+
)
|
|
363
|
+
|
|
364
|
+
def postfix(self, rh, opts):
|
|
365
|
+
"""Find out if any special resources have been produced."""
|
|
366
|
+
sh = self.system
|
|
367
|
+
# Gather rawfiles in folders
|
|
368
|
+
if self.rawfiles:
|
|
369
|
+
flist = sh.glob('tmprawfile_D000_L*')
|
|
370
|
+
dest = 'rawfiles'
|
|
371
|
+
logger.info('Creating a rawfiles pack: %s', dest)
|
|
372
|
+
sh.mkdir(dest)
|
|
373
|
+
for fic in flist:
|
|
374
|
+
sh.mv(fic, dest, fmt='grib')
|
|
375
|
+
super().postfix(rh, opts)
|
|
376
|
+
|
|
377
|
+
|
|
378
|
+
class IFSInflationLike(IFSEdaAbstractAlgo):
|
|
379
|
+
"""Apply the inflation scheme on a given modelstate."""
|
|
380
|
+
|
|
381
|
+
_RUNSTORE = 'RUNOUT'
|
|
382
|
+
_USELESS_MATCH = re.compile(r'^(?P<target>\w+)\+term\d+:\d+$')
|
|
383
|
+
|
|
384
|
+
_footprint = dict(
|
|
385
|
+
info='Operations around the background error covariance matrix',
|
|
386
|
+
attr=dict(
|
|
387
|
+
kind=dict(
|
|
388
|
+
values=['infl', 'pert', ],
|
|
389
|
+
),
|
|
390
|
+
conf=dict(
|
|
391
|
+
values=[701, ]
|
|
392
|
+
)
|
|
393
|
+
)
|
|
394
|
+
)
|
|
395
|
+
|
|
396
|
+
def __init__(self, *kargs, **kwargs):
|
|
397
|
+
super().__init__(*kargs, **kwargs)
|
|
398
|
+
self._outputs_shelf = list()
|
|
399
|
+
|
|
400
|
+
def _check_effective_terms(self, roles):
|
|
401
|
+
eff_terms = None
|
|
402
|
+
for role in roles:
|
|
403
|
+
eterm = {sec.rh.resource.term
|
|
404
|
+
for sec in self.context.sequence.effective_inputs(role=role)}
|
|
405
|
+
if eterm:
|
|
406
|
+
if eff_terms is None:
|
|
407
|
+
eff_terms = eterm
|
|
408
|
+
else:
|
|
409
|
+
if eff_terms != eterm:
|
|
410
|
+
raise AlgoComponentError("Inconsistencies between inputs effective terms.")
|
|
411
|
+
return sorted(eff_terms)
|
|
412
|
+
|
|
413
|
+
def _link_stuff_in(self, role, actualterm, targetnc, targetintent='in', wastebasket=None):
|
|
414
|
+
estuff = [sec
|
|
415
|
+
for sec in self.context.sequence.effective_inputs(role=role)
|
|
416
|
+
if sec.rh.resource.term == actualterm]
|
|
417
|
+
if len(estuff) > 1:
|
|
418
|
+
logger.warning('Multiple %s for the same date ! Going on...', role)
|
|
419
|
+
elif len(estuff) == 1:
|
|
420
|
+
# Detect the inputs format
|
|
421
|
+
actfmt = estuff[0].rh.container.actualfmt
|
|
422
|
+
nconv = self.naming_convention(actualfmt=actfmt, **targetnc)
|
|
423
|
+
targetname = nconv(**targetnc)
|
|
424
|
+
if self.system.path.exists(targetname):
|
|
425
|
+
logger.info("%s: %s already exists. Hopping for the best...",
|
|
426
|
+
role, targetname)
|
|
427
|
+
else:
|
|
428
|
+
logger.info("%s: copying (intent=%s) %s to %s", role, targetintent,
|
|
429
|
+
estuff[0].rh.container.localpath(), targetname)
|
|
430
|
+
self.system.cp(estuff[0].rh.container.localpath(), targetname,
|
|
431
|
+
fmt=actfmt, intent=targetintent)
|
|
432
|
+
if wastebasket is not None:
|
|
433
|
+
wastebasket.append((targetname, actfmt))
|
|
434
|
+
return nconv, targetname, estuff[0]
|
|
435
|
+
return None, None, None
|
|
436
|
+
|
|
437
|
+
def execute(self, rh, opts):
|
|
438
|
+
"""Loop on the various terms provided."""
|
|
439
|
+
|
|
440
|
+
eff_terms = self._check_effective_terms(['ModelState',
|
|
441
|
+
'EnsembleMean',
|
|
442
|
+
'Guess'])
|
|
443
|
+
fix_curclim = self.do_climfile_fixer(rh, convkind='modelclim')
|
|
444
|
+
fix_clclim = self.do_climfile_fixer(rh, convkind='closest_modelclim')
|
|
445
|
+
|
|
446
|
+
if eff_terms:
|
|
447
|
+
for actualterm in eff_terms:
|
|
448
|
+
wastebasket = list()
|
|
449
|
+
self.system.title('Loop on term {!s}'.format(actualterm))
|
|
450
|
+
self.system.subtitle('Solving the input files nightmare...')
|
|
451
|
+
# Ensemble Mean ?
|
|
452
|
+
mean_number = 2 if self.model == 'arome' else 0
|
|
453
|
+
targetnc = dict(kind='edainput', variant='infl', rh=rh, number=mean_number)
|
|
454
|
+
self._link_stuff_in('EnsembleMean', actualterm, targetnc,
|
|
455
|
+
wastebasket=wastebasket)
|
|
456
|
+
# Model State ?
|
|
457
|
+
targetnc = dict(kind='edainput', variant='infl', rh=rh, number=1)
|
|
458
|
+
_, _, mstate = self._link_stuff_in('ModelState', actualterm, targetnc,
|
|
459
|
+
wastebasket=wastebasket)
|
|
460
|
+
# Control ?
|
|
461
|
+
control_number = 0 if self.model == 'arome' else 2
|
|
462
|
+
targetnc = dict(kind='edainput', variant='infl', rh=rh, number=control_number)
|
|
463
|
+
self._link_stuff_in('Control', actualterm, targetnc,
|
|
464
|
+
wastebasket=wastebasket)
|
|
465
|
+
# Guess ?
|
|
466
|
+
targetnc = dict(kind='edaoutput', variant='infl', rh=rh, number=1, term=Time(0))
|
|
467
|
+
outnc, _, _ = self._link_stuff_in('Guess', actualterm, targetnc,
|
|
468
|
+
targetintent='inout')
|
|
469
|
+
if outnc is None:
|
|
470
|
+
outnc = self.naming_convention(kind='edaoutput', variant='infl', rh=rh)
|
|
471
|
+
# Fix clim !
|
|
472
|
+
if fix_curclim and mstate:
|
|
473
|
+
month = Month((mstate.rh.resource.date + actualterm).ymdh)
|
|
474
|
+
self.climfile_fixer(rh=rh, convkind='modelclim', month=month,
|
|
475
|
+
inputrole=('GlobalClim', 'InitialClim'),
|
|
476
|
+
inputkind='clim_model')
|
|
477
|
+
if fix_clclim and mstate:
|
|
478
|
+
closestmonth = Month((mstate.rh.resource.date + actualterm).ymdh + ':closest')
|
|
479
|
+
self.climfile_fixer(rh=rh, convkind='closest_modelclim', month=closestmonth,
|
|
480
|
+
inputrole=('GlobalClim', 'InitialClim'),
|
|
481
|
+
inputkind='clim_model')
|
|
482
|
+
# Deal with useless stuff... SADLY !
|
|
483
|
+
useless = [sec
|
|
484
|
+
for sec in self.context.sequence.effective_inputs(role='Useless')
|
|
485
|
+
if (sec.rh.resource.term == actualterm and
|
|
486
|
+
self._USELESS_MATCH.match(sec.rh.container.localpath()))]
|
|
487
|
+
for a_useless in useless:
|
|
488
|
+
targetname = self._USELESS_MATCH.match(a_useless.rh.container.localpath()).group('target')
|
|
489
|
+
if self.system.path.exists(targetname):
|
|
490
|
+
logger.warning("Some useless stuff is already here: %s. I don't care...",
|
|
491
|
+
targetname)
|
|
492
|
+
else:
|
|
493
|
+
logger.info("Dealing with useless stuff: %s -> %s",
|
|
494
|
+
a_useless.rh.container.localpath(), targetname)
|
|
495
|
+
self.system.cp(a_useless.rh.container.localpath(), targetname,
|
|
496
|
+
fmt=a_useless.rh.container.actualfmt, intent='in')
|
|
497
|
+
wastebasket.append((targetname, a_useless.rh.container.actualfmt))
|
|
498
|
+
|
|
499
|
+
# Standard execution
|
|
500
|
+
super().execute(rh, opts)
|
|
501
|
+
|
|
502
|
+
# The concatenated listing
|
|
503
|
+
self.system.cat('NODE.001_01', output='NODE.all')
|
|
504
|
+
|
|
505
|
+
# prepares the next execution
|
|
506
|
+
if len(eff_terms) > 1:
|
|
507
|
+
self.system.mkdir(self._RUNSTORE)
|
|
508
|
+
# Freeze the current output
|
|
509
|
+
shelf_label = self.system.path.join(self._RUNSTORE, outnc(number=1, term=actualterm))
|
|
510
|
+
self.system.move(outnc(number=1, term=Time(0)), shelf_label, fmt='fa')
|
|
511
|
+
self._outputs_shelf.append(shelf_label)
|
|
512
|
+
# Some cleaning
|
|
513
|
+
for afile in wastebasket:
|
|
514
|
+
self.system.remove(afile[0], fmt=afile[1])
|
|
515
|
+
self.system.rmall('ncf927', 'dirlst')
|
|
516
|
+
else:
|
|
517
|
+
# We should not be here but whatever... some task are poorly written !
|
|
518
|
+
super().execute(rh, opts)
|
|
519
|
+
|
|
520
|
+
def postfix(self, rh, opts):
|
|
521
|
+
"""Post-processing cleaning."""
|
|
522
|
+
self.system.title('Finalising the execution...')
|
|
523
|
+
for afile in self._outputs_shelf:
|
|
524
|
+
logger.info("Output found: %s", self.system.path.basename(afile))
|
|
525
|
+
self.system.move(afile, self.system.path.basename(afile), fmt='fa')
|
|
526
|
+
super().postfix(rh, opts)
|
|
527
|
+
|
|
528
|
+
|
|
529
|
+
class IFSInflationFactor(IFSEdaEnsembleAbstractAlgo):
|
|
530
|
+
"""Compute an inflation factor based on individual members."""
|
|
531
|
+
|
|
532
|
+
_footprint = dict(
|
|
533
|
+
info='Compute an inflation factor based on individual members',
|
|
534
|
+
attr=dict(
|
|
535
|
+
kind=dict(
|
|
536
|
+
values=['infl_factor', ],
|
|
537
|
+
),
|
|
538
|
+
)
|
|
539
|
+
)
|
|
540
|
+
|
|
541
|
+
|
|
542
|
+
class IFSInflationFactorLegacy(IFSInflationFactor):
|
|
543
|
+
"""Compute an inflation factor based on individual members. KEPT FOR COMPATIBILITY.
|
|
544
|
+
|
|
545
|
+
DO NOT USE !
|
|
546
|
+
"""
|
|
547
|
+
|
|
548
|
+
_footprint = dict(
|
|
549
|
+
info='Compute an inflation factor based on individual members',
|
|
550
|
+
attr=dict(
|
|
551
|
+
kind=dict(
|
|
552
|
+
values=['infl', 'pert'],
|
|
553
|
+
),
|
|
554
|
+
conf=dict(
|
|
555
|
+
outcast=[701, ]
|
|
556
|
+
)
|
|
557
|
+
)
|
|
558
|
+
)
|
|
559
|
+
|
|
560
|
+
|
|
561
|
+
class IFSEnsembleMean(IFSEdaEnsembleAbstractAlgo):
|
|
562
|
+
"""Apply the inflation scheme on a given modelstate."""
|
|
563
|
+
|
|
564
|
+
_footprint = dict(
|
|
565
|
+
info='Operations around the background error covariance matrix',
|
|
566
|
+
attr=dict(
|
|
567
|
+
kind=dict(
|
|
568
|
+
values=['mean', ],
|
|
569
|
+
),
|
|
570
|
+
)
|
|
571
|
+
)
|
|
572
|
+
|
|
573
|
+
|
|
574
|
+
class IFSCovB(IFSEdaLaggedEnsembleAbstractAlgo):
|
|
575
|
+
"""Operations around the background error covariance matrix."""
|
|
576
|
+
|
|
577
|
+
_footprint = dict(
|
|
578
|
+
info='Operations around the background error covariance matrix',
|
|
579
|
+
attr=dict(
|
|
580
|
+
kind=dict(
|
|
581
|
+
values=['covb', ],
|
|
582
|
+
),
|
|
583
|
+
hybrid = dict(
|
|
584
|
+
type = bool,
|
|
585
|
+
optional = True,
|
|
586
|
+
default = False,
|
|
587
|
+
),
|
|
588
|
+
)
|
|
589
|
+
)
|
|
590
|
+
|
|
591
|
+
_HYBRID_CLIM_ROLE = 'ClimatologicalModelState'
|
|
592
|
+
|
|
593
|
+
@property
|
|
594
|
+
def actual_totalnumber(self):
|
|
595
|
+
"""The total number of members (times 2 if hybrid...)."""
|
|
596
|
+
parent_totalnumber = super().actual_totalnumber
|
|
597
|
+
return parent_totalnumber * 2 if self.hybrid else parent_totalnumber
|
|
598
|
+
|
|
599
|
+
def prepare(self, rh, opts):
|
|
600
|
+
"""Default pre-link for the initial condition file"""
|
|
601
|
+
super().prepare(rh, opts)
|
|
602
|
+
# Legacy...
|
|
603
|
+
for num, sec in enumerate(sorted(self.context.sequence.effective_inputs(role='Rawfiles'),
|
|
604
|
+
key=self._members_sorting_key),
|
|
605
|
+
start=1):
|
|
606
|
+
repname = sec.rh.container.localpath()
|
|
607
|
+
radical = repname.split('_')[0] + '_D{:03d}_L{:s}'
|
|
608
|
+
for filename in self.system.listdir(repname):
|
|
609
|
+
level = re.search(r'_L(\d+)$', filename)
|
|
610
|
+
if level is not None:
|
|
611
|
+
self.system.softlink(self.system.path.join(repname, filename),
|
|
612
|
+
radical.format(num, level.group(1)))
|
|
613
|
+
# Legacy...
|
|
614
|
+
for num, sec in enumerate(sorted(self.context.sequence.effective_inputs(role='LaggedEnsemble'),
|
|
615
|
+
key=self._members_sorting_key),
|
|
616
|
+
start=1):
|
|
617
|
+
repname = sec.rh.container.localpath()
|
|
618
|
+
radical = repname.split('_')[0] + '_{:03d}'
|
|
619
|
+
self.system.softlink(repname, radical.format(num))
|
|
620
|
+
# Requesting Hybrid computations ?
|
|
621
|
+
if self.hybrid:
|
|
622
|
+
hybstuff = self.context.sequence.effective_inputs(role=self._HYBRID_CLIM_ROLE)
|
|
623
|
+
hybformat = hybstuff[0].rh.container.actualfmt
|
|
624
|
+
totalnumber = self.actual_nbe * self.actual_nresx if self.padding else self.actual_nbe
|
|
625
|
+
for i, tnum in enumerate(range(totalnumber + 1, 2 * totalnumber + 1)):
|
|
626
|
+
innc = self.naming_convention(kind='edainput', variant=self.kind,
|
|
627
|
+
totalnumber=self.actual_totalnumber,
|
|
628
|
+
rh=rh, actualfmt=hybformat)
|
|
629
|
+
logger.info("Soft-Linking %s to %s",
|
|
630
|
+
hybstuff[i].rh.container.localpath(), innc(number=tnum))
|
|
631
|
+
self.system.softlink(hybstuff[i].rh.container.localpath(),
|
|
632
|
+
innc(number=tnum))
|