vortex-nwp 2.0.0b1__py3-none-any.whl → 2.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- vortex/__init__.py +75 -47
- vortex/algo/__init__.py +3 -2
- vortex/algo/components.py +944 -618
- vortex/algo/mpitools.py +802 -497
- vortex/algo/mpitools_templates/__init__.py +1 -0
- vortex/algo/serversynctools.py +34 -33
- vortex/config.py +19 -22
- vortex/data/__init__.py +9 -3
- vortex/data/abstractstores.py +593 -655
- vortex/data/containers.py +217 -162
- vortex/data/contents.py +65 -39
- vortex/data/executables.py +93 -102
- vortex/data/flow.py +40 -34
- vortex/data/geometries.py +228 -132
- vortex/data/handlers.py +436 -227
- vortex/data/outflow.py +15 -15
- vortex/data/providers.py +185 -163
- vortex/data/resources.py +48 -42
- vortex/data/stores.py +540 -417
- vortex/data/sync_templates/__init__.py +0 -0
- vortex/gloves.py +114 -87
- vortex/layout/__init__.py +1 -8
- vortex/layout/contexts.py +150 -84
- vortex/layout/dataflow.py +353 -202
- vortex/layout/monitor.py +264 -128
- vortex/nwp/__init__.py +5 -2
- vortex/nwp/algo/__init__.py +14 -5
- vortex/nwp/algo/assim.py +205 -151
- vortex/nwp/algo/clim.py +683 -517
- vortex/nwp/algo/coupling.py +447 -225
- vortex/nwp/algo/eda.py +437 -229
- vortex/nwp/algo/eps.py +403 -231
- vortex/nwp/algo/forecasts.py +416 -275
- vortex/nwp/algo/fpserver.py +683 -307
- vortex/nwp/algo/ifsnaming.py +205 -145
- vortex/nwp/algo/ifsroot.py +215 -122
- vortex/nwp/algo/monitoring.py +137 -76
- vortex/nwp/algo/mpitools.py +330 -190
- vortex/nwp/algo/odbtools.py +637 -353
- vortex/nwp/algo/oopsroot.py +454 -273
- vortex/nwp/algo/oopstests.py +90 -56
- vortex/nwp/algo/request.py +287 -206
- vortex/nwp/algo/stdpost.py +878 -522
- vortex/nwp/data/__init__.py +22 -4
- vortex/nwp/data/assim.py +125 -137
- vortex/nwp/data/boundaries.py +121 -68
- vortex/nwp/data/climfiles.py +193 -211
- vortex/nwp/data/configfiles.py +73 -69
- vortex/nwp/data/consts.py +426 -401
- vortex/nwp/data/ctpini.py +59 -43
- vortex/nwp/data/diagnostics.py +94 -66
- vortex/nwp/data/eda.py +50 -51
- vortex/nwp/data/eps.py +195 -146
- vortex/nwp/data/executables.py +440 -434
- vortex/nwp/data/fields.py +63 -48
- vortex/nwp/data/gridfiles.py +183 -111
- vortex/nwp/data/logs.py +250 -217
- vortex/nwp/data/modelstates.py +180 -151
- vortex/nwp/data/monitoring.py +72 -99
- vortex/nwp/data/namelists.py +254 -202
- vortex/nwp/data/obs.py +400 -308
- vortex/nwp/data/oopsexec.py +22 -20
- vortex/nwp/data/providers.py +90 -65
- vortex/nwp/data/query.py +71 -82
- vortex/nwp/data/stores.py +49 -36
- vortex/nwp/data/surfex.py +136 -137
- vortex/nwp/syntax/__init__.py +1 -1
- vortex/nwp/syntax/stdattrs.py +173 -111
- vortex/nwp/tools/__init__.py +2 -2
- vortex/nwp/tools/addons.py +22 -17
- vortex/nwp/tools/agt.py +24 -12
- vortex/nwp/tools/bdap.py +16 -5
- vortex/nwp/tools/bdcp.py +4 -1
- vortex/nwp/tools/bdm.py +3 -0
- vortex/nwp/tools/bdmp.py +14 -9
- vortex/nwp/tools/conftools.py +728 -378
- vortex/nwp/tools/drhook.py +12 -8
- vortex/nwp/tools/grib.py +65 -39
- vortex/nwp/tools/gribdiff.py +22 -17
- vortex/nwp/tools/ifstools.py +82 -42
- vortex/nwp/tools/igastuff.py +167 -143
- vortex/nwp/tools/mars.py +14 -2
- vortex/nwp/tools/odb.py +234 -125
- vortex/nwp/tools/partitioning.py +61 -37
- vortex/nwp/tools/satrad.py +27 -12
- vortex/nwp/util/async.py +83 -55
- vortex/nwp/util/beacon.py +10 -10
- vortex/nwp/util/diffpygram.py +174 -86
- vortex/nwp/util/ens.py +144 -63
- vortex/nwp/util/hooks.py +30 -19
- vortex/nwp/util/taskdeco.py +28 -24
- vortex/nwp/util/usepygram.py +278 -172
- vortex/nwp/util/usetnt.py +31 -17
- vortex/sessions.py +72 -39
- vortex/syntax/__init__.py +1 -1
- vortex/syntax/stdattrs.py +410 -171
- vortex/syntax/stddeco.py +31 -22
- vortex/toolbox.py +327 -192
- vortex/tools/__init__.py +11 -2
- vortex/tools/actions.py +110 -121
- vortex/tools/addons.py +111 -92
- vortex/tools/arm.py +42 -22
- vortex/tools/compression.py +72 -69
- vortex/tools/date.py +11 -4
- vortex/tools/delayedactions.py +242 -132
- vortex/tools/env.py +75 -47
- vortex/tools/folder.py +342 -171
- vortex/tools/grib.py +341 -162
- vortex/tools/lfi.py +423 -216
- vortex/tools/listings.py +109 -40
- vortex/tools/names.py +218 -156
- vortex/tools/net.py +655 -299
- vortex/tools/parallelism.py +93 -61
- vortex/tools/prestaging.py +55 -31
- vortex/tools/schedulers.py +172 -105
- vortex/tools/services.py +403 -334
- vortex/tools/storage.py +293 -358
- vortex/tools/surfex.py +24 -24
- vortex/tools/systems.py +1234 -643
- vortex/tools/targets.py +156 -100
- vortex/util/__init__.py +1 -1
- vortex/util/config.py +378 -327
- vortex/util/empty.py +2 -2
- vortex/util/helpers.py +56 -24
- vortex/util/introspection.py +18 -12
- vortex/util/iosponge.py +8 -4
- vortex/util/roles.py +4 -6
- vortex/util/storefunctions.py +39 -13
- vortex/util/structs.py +3 -3
- vortex/util/worker.py +29 -17
- vortex_nwp-2.1.0.dist-info/METADATA +67 -0
- vortex_nwp-2.1.0.dist-info/RECORD +144 -0
- {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.1.0.dist-info}/WHEEL +1 -1
- vortex/layout/appconf.py +0 -109
- vortex/layout/jobs.py +0 -1276
- vortex/layout/nodes.py +0 -1424
- vortex/layout/subjobs.py +0 -464
- vortex_nwp-2.0.0b1.dist-info/METADATA +0 -50
- vortex_nwp-2.0.0b1.dist-info/RECORD +0 -146
- {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.1.0.dist-info/licenses}/LICENSE +0 -0
- {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.1.0.dist-info}/top_level.txt +0 -0
vortex/nwp/algo/fpserver.py
CHANGED
|
@@ -29,7 +29,7 @@ __all__ = []
|
|
|
29
29
|
logger = loggers.getLogger(__name__)
|
|
30
30
|
|
|
31
31
|
|
|
32
|
-
fullpos_server_flypoll_pickle =
|
|
32
|
+
fullpos_server_flypoll_pickle = ".fullpos_server_flypoll"
|
|
33
33
|
|
|
34
34
|
|
|
35
35
|
class FullPosServerFlyPollPersistantState:
|
|
@@ -40,7 +40,9 @@ class FullPosServerFlyPollPersistantState:
|
|
|
40
40
|
self.found = collections.defaultdict(list)
|
|
41
41
|
|
|
42
42
|
|
|
43
|
-
def fullpos_server_flypoll(
|
|
43
|
+
def fullpos_server_flypoll(
|
|
44
|
+
sh, outputprefix, termfile, directories=(".",), **kwargs
|
|
45
|
+
): # @UnusedVariable
|
|
44
46
|
"""Check sub-**directories** to determine wether new output files are available or not."""
|
|
45
47
|
new = list()
|
|
46
48
|
for directory in directories:
|
|
@@ -52,25 +54,41 @@ def fullpos_server_flypoll(sh, outputprefix, termfile, directories=('.', ), **kw
|
|
|
52
54
|
try:
|
|
53
55
|
if sh.path.exists(termfile):
|
|
54
56
|
with open(termfile) as wfh:
|
|
55
|
-
rawcursor = wfh.readline().rstrip(
|
|
57
|
+
rawcursor = wfh.readline().rstrip("\n")
|
|
56
58
|
try:
|
|
57
59
|
cursor = Time(rawcursor)
|
|
58
60
|
except TypeError:
|
|
59
|
-
logger.warning(
|
|
61
|
+
logger.warning(
|
|
62
|
+
'Unable to convert "%s" to a Time object',
|
|
63
|
+
rawcursor,
|
|
64
|
+
)
|
|
60
65
|
return new
|
|
61
|
-
pre = re.compile(
|
|
66
|
+
pre = re.compile(
|
|
67
|
+
r"^{:s}\w*\+(\d+(?::\d\d)?)(?:\.\w+)?$".format(
|
|
68
|
+
outputprefix
|
|
69
|
+
)
|
|
70
|
+
)
|
|
62
71
|
candidates = [pre.match(f) for f in sh.listdir()]
|
|
63
72
|
lnew = list()
|
|
64
|
-
for candidate in filterfalse(
|
|
65
|
-
|
|
73
|
+
for candidate in filterfalse(
|
|
74
|
+
lambda c: c is None, candidates
|
|
75
|
+
):
|
|
76
|
+
if candidate.group(0).endswith(".d"):
|
|
66
77
|
continue
|
|
67
78
|
ctime = Time(candidate.group(1))
|
|
68
|
-
if
|
|
79
|
+
if (
|
|
80
|
+
ctime > fpoll_st.cursor[outputprefix]
|
|
81
|
+
and ctime <= cursor
|
|
82
|
+
):
|
|
69
83
|
lnew.append(candidate.group(0))
|
|
70
84
|
fpoll_st.cursor[outputprefix] = cursor
|
|
71
85
|
fpoll_st.found[outputprefix].extend(lnew)
|
|
72
|
-
new.extend(
|
|
73
|
-
|
|
86
|
+
new.extend(
|
|
87
|
+
[
|
|
88
|
+
sh.path.normpath(sh.path.join(directory, anew))
|
|
89
|
+
for anew in lnew
|
|
90
|
+
]
|
|
91
|
+
)
|
|
74
92
|
finally:
|
|
75
93
|
sh.pickle_dump(fpoll_st, fullpos_server_flypoll_pickle)
|
|
76
94
|
return new
|
|
@@ -92,8 +110,7 @@ class FullposServerDiscoveredInputs:
|
|
|
92
110
|
"""Find out the required suffixlen."""
|
|
93
111
|
if minlen is None:
|
|
94
112
|
minlen = self.inputsminlen
|
|
95
|
-
return max(minlen,
|
|
96
|
-
int(math.floor(math.log10(len(self.tododata)))))
|
|
113
|
+
return max(minlen, int(math.floor(math.log10(len(self.tododata)))))
|
|
97
114
|
|
|
98
115
|
|
|
99
116
|
class FullPosServer(IFSParallel):
|
|
@@ -141,16 +158,16 @@ class FullPosServer(IFSParallel):
|
|
|
141
158
|
|
|
142
159
|
"""
|
|
143
160
|
|
|
144
|
-
_INITIALCONDITION_ROLE = re.compile(r
|
|
145
|
-
_INPUTDATA_ROLE_STR =
|
|
146
|
-
_INPUTDATA_ROLE = re.compile(r
|
|
147
|
-
_OUTPUTGUESS_ROLE =
|
|
161
|
+
_INITIALCONDITION_ROLE = re.compile(r"InitialCondition((?:\w+)?)")
|
|
162
|
+
_INPUTDATA_ROLE_STR = "ModelState"
|
|
163
|
+
_INPUTDATA_ROLE = re.compile(r"ModelState((?:\w+)?)")
|
|
164
|
+
_OUTPUTGUESS_ROLE = "OutputGuess"
|
|
148
165
|
|
|
149
|
-
_MODELSIDE_INPUTPREFIX0 =
|
|
150
|
-
_MODELSIDE_INPUTPREFIX1 =
|
|
151
|
-
_MODELSIDE_OUTPUTPREFIX =
|
|
152
|
-
_MODELSIDE_OUTPUTPREFIX_GRIB =
|
|
153
|
-
_MODELSIDE_TERMFILE =
|
|
166
|
+
_MODELSIDE_INPUTPREFIX0 = "ICM"
|
|
167
|
+
_MODELSIDE_INPUTPREFIX1 = "SH"
|
|
168
|
+
_MODELSIDE_OUTPUTPREFIX = "PF"
|
|
169
|
+
_MODELSIDE_OUTPUTPREFIX_GRIB = "GRIBPF"
|
|
170
|
+
_MODELSIDE_TERMFILE = "./ECHFP"
|
|
154
171
|
_MODELSIDE_OUT_SUFFIXLEN_MIN = 4
|
|
155
172
|
_MODELSIDE_IND_SUFFIXLEN_MIN = 4
|
|
156
173
|
_MODELSIDE_INE_SUFFIXLEN_MIN = dict(grib=6)
|
|
@@ -162,77 +179,83 @@ class FullPosServer(IFSParallel):
|
|
|
162
179
|
_footprint = [
|
|
163
180
|
outputid_deco,
|
|
164
181
|
dict(
|
|
165
|
-
attr
|
|
166
|
-
kind
|
|
167
|
-
values
|
|
182
|
+
attr=dict(
|
|
183
|
+
kind=dict(
|
|
184
|
+
values=[
|
|
185
|
+
"fpserver",
|
|
186
|
+
],
|
|
168
187
|
),
|
|
169
|
-
outdirectories
|
|
170
|
-
info
|
|
171
|
-
type
|
|
172
|
-
default
|
|
173
|
-
|
|
188
|
+
outdirectories=dict(
|
|
189
|
+
info="The list of possible output directories.",
|
|
190
|
+
type=footprints.stdtypes.FPList,
|
|
191
|
+
default=footprints.stdtypes.FPList(
|
|
192
|
+
[
|
|
193
|
+
".",
|
|
194
|
+
]
|
|
195
|
+
),
|
|
196
|
+
optional=True,
|
|
174
197
|
),
|
|
175
|
-
append_domain
|
|
176
|
-
info
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
198
|
+
append_domain=dict(
|
|
199
|
+
info=(
|
|
200
|
+
"If defined, the output file for domain append_domain "
|
|
201
|
+
+ "will be made a copy of the input file (prior to the "
|
|
202
|
+
+ "server run"
|
|
203
|
+
),
|
|
204
|
+
optional=True,
|
|
180
205
|
),
|
|
181
206
|
basedate=dict(
|
|
182
|
-
info
|
|
183
|
-
type
|
|
184
|
-
optional
|
|
207
|
+
info="The run date of the coupling generating process",
|
|
208
|
+
type=Date,
|
|
209
|
+
optional=True,
|
|
185
210
|
),
|
|
186
|
-
xpname
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
conf = dict(
|
|
190
|
-
default = 903,
|
|
211
|
+
xpname=dict(default="FPOS"),
|
|
212
|
+
conf=dict(
|
|
213
|
+
default=903,
|
|
191
214
|
),
|
|
192
215
|
timestep=dict(
|
|
193
|
-
default
|
|
216
|
+
default=1.0,
|
|
194
217
|
),
|
|
195
|
-
timeout
|
|
196
|
-
type
|
|
197
|
-
optional
|
|
198
|
-
default
|
|
218
|
+
timeout=dict(
|
|
219
|
+
type=int,
|
|
220
|
+
optional=True,
|
|
221
|
+
default=300,
|
|
199
222
|
),
|
|
200
|
-
refreshtime
|
|
201
|
-
info
|
|
202
|
-
type
|
|
203
|
-
optional
|
|
204
|
-
default
|
|
223
|
+
refreshtime=dict(
|
|
224
|
+
info="How frequently are the expected input files looked for ? (seconds)",
|
|
225
|
+
type=int,
|
|
226
|
+
optional=True,
|
|
227
|
+
default=20,
|
|
205
228
|
),
|
|
206
|
-
server_run
|
|
229
|
+
server_run=dict(
|
|
207
230
|
# This is a rw attribute: it will be managed internally
|
|
208
|
-
values
|
|
231
|
+
values=[True, False]
|
|
209
232
|
),
|
|
210
|
-
serversync_method
|
|
211
|
-
default
|
|
233
|
+
serversync_method=dict(
|
|
234
|
+
default="simple_socket",
|
|
212
235
|
),
|
|
213
|
-
serversync_medium
|
|
214
|
-
default
|
|
236
|
+
serversync_medium=dict(
|
|
237
|
+
default="nextfile_wait",
|
|
215
238
|
),
|
|
216
|
-
maxpollingthreads
|
|
217
|
-
type
|
|
218
|
-
optional
|
|
219
|
-
default
|
|
239
|
+
maxpollingthreads=dict(
|
|
240
|
+
type=int,
|
|
241
|
+
optional=True,
|
|
242
|
+
default=8,
|
|
220
243
|
),
|
|
221
|
-
flypoll
|
|
222
|
-
default
|
|
244
|
+
flypoll=dict(
|
|
245
|
+
default="internal",
|
|
246
|
+
),
|
|
247
|
+
defaultformat=dict(
|
|
248
|
+
info="Format for the legacy output files.",
|
|
249
|
+
default="fa",
|
|
250
|
+
optional=True,
|
|
223
251
|
),
|
|
224
|
-
defaultformat = dict(
|
|
225
|
-
info = "Format for the legacy output files.",
|
|
226
|
-
default = 'fa',
|
|
227
|
-
optional = True
|
|
228
|
-
)
|
|
229
252
|
)
|
|
230
|
-
)
|
|
253
|
+
),
|
|
231
254
|
]
|
|
232
255
|
|
|
233
256
|
@property
|
|
234
257
|
def realkind(self):
|
|
235
|
-
return
|
|
258
|
+
return "fullpos"
|
|
236
259
|
|
|
237
260
|
def __init__(self, *args, **kw):
|
|
238
261
|
super().__init__(*args, **kw)
|
|
@@ -244,10 +267,15 @@ class FullPosServer(IFSParallel):
|
|
|
244
267
|
for out_re, data in self._flyput_mapping_d.items():
|
|
245
268
|
m_re = out_re.match(sh.path.basename(item))
|
|
246
269
|
if m_re:
|
|
247
|
-
return (
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
data[
|
|
270
|
+
return (
|
|
271
|
+
sh.path.join(
|
|
272
|
+
sh.path.dirname(item),
|
|
273
|
+
data[0].format(
|
|
274
|
+
m_re.group("fpdom"), m_re.group("suffix")
|
|
275
|
+
),
|
|
276
|
+
),
|
|
277
|
+
data[1],
|
|
278
|
+
)
|
|
251
279
|
|
|
252
280
|
@cached_property
|
|
253
281
|
def inputs(self):
|
|
@@ -255,108 +283,174 @@ class FullPosServer(IFSParallel):
|
|
|
255
283
|
discovered = FullposServerDiscoveredInputs()
|
|
256
284
|
|
|
257
285
|
# Initial conditions
|
|
258
|
-
inisec = self.context.sequence.effective_inputs(
|
|
286
|
+
inisec = self.context.sequence.effective_inputs(
|
|
287
|
+
role=self._INITIALCONDITION_ROLE
|
|
288
|
+
)
|
|
259
289
|
if inisec:
|
|
260
290
|
for s in inisec:
|
|
261
|
-
iprefix = (
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
291
|
+
iprefix = (
|
|
292
|
+
self._INITIALCONDITION_ROLE.match(
|
|
293
|
+
s.alternate if s.role is None else s.role
|
|
294
|
+
).group(1)
|
|
295
|
+
or self._MODELSIDE_INPUTPREFIX1
|
|
296
|
+
)
|
|
265
297
|
fprefix = self._MODELSIDE_INPUTPREFIX0 + iprefix
|
|
266
298
|
if fprefix in discovered.inidata:
|
|
267
|
-
raise AlgoComponentError(
|
|
299
|
+
raise AlgoComponentError(
|
|
300
|
+
"Only one Initial Condition is allowed."
|
|
301
|
+
)
|
|
268
302
|
else:
|
|
269
303
|
discovered.inidata[fprefix] = s
|
|
270
304
|
|
|
271
305
|
# Model states
|
|
272
|
-
todosec0 = self.context.sequence.effective_inputs(
|
|
306
|
+
todosec0 = self.context.sequence.effective_inputs(
|
|
307
|
+
role=self._INPUTDATA_ROLE
|
|
308
|
+
)
|
|
273
309
|
todosec1 = collections.defaultdict(list)
|
|
274
|
-
discovered.anyexpected = any(
|
|
275
|
-
|
|
310
|
+
discovered.anyexpected = any(
|
|
311
|
+
[isec.rh.is_expected() for isec in todosec0]
|
|
312
|
+
)
|
|
313
|
+
hasterms = all(
|
|
314
|
+
[hasattr(isec.rh.resource, "term") for isec in todosec0]
|
|
315
|
+
)
|
|
276
316
|
# Sort things up (if possible)
|
|
277
317
|
if hasterms:
|
|
278
|
-
logger.info(
|
|
318
|
+
logger.info("Sorting input data based on the actual term.")
|
|
279
319
|
todosec0 = sorted(todosec0, key=lambda s: self._actual_term(s.rh))
|
|
280
320
|
if todosec0:
|
|
281
321
|
for iseq, s in enumerate(todosec0):
|
|
282
|
-
rprefix = (
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
322
|
+
rprefix = (
|
|
323
|
+
self._INPUTDATA_ROLE.match(
|
|
324
|
+
s.alternate if s.role is None else s.role
|
|
325
|
+
).group(1)
|
|
326
|
+
or self._MODELSIDE_INPUTPREFIX1
|
|
327
|
+
)
|
|
286
328
|
todosec1[rprefix].append(s)
|
|
287
329
|
if iseq == 0:
|
|
288
330
|
# Find the "default" prefix and suffix len based on the first section
|
|
289
331
|
discovered.firstprefix = rprefix
|
|
290
|
-
discovered.inputsminlen =
|
|
291
|
-
|
|
332
|
+
discovered.inputsminlen = (
|
|
333
|
+
self._MODELSIDE_INE_SUFFIXLEN_MIN.get(
|
|
334
|
+
s.rh.container.actualfmt,
|
|
335
|
+
self._MODELSIDE_IND_SUFFIXLEN_MIN,
|
|
336
|
+
)
|
|
292
337
|
)
|
|
293
338
|
iprefixes = sorted(todosec1.keys())
|
|
294
339
|
if len(iprefixes) == 1:
|
|
295
340
|
for s in todosec0:
|
|
296
|
-
discovered.tododata.append(
|
|
341
|
+
discovered.tododata.append(
|
|
342
|
+
{self._MODELSIDE_INPUTPREFIX0 + iprefixes[0]: s}
|
|
343
|
+
)
|
|
297
344
|
else:
|
|
298
345
|
if len({len(secs) for secs in todosec1.values()}) > 1:
|
|
299
|
-
raise AlgoComponentError(
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
346
|
+
raise AlgoComponentError(
|
|
347
|
+
"Inconsistent number of input data."
|
|
348
|
+
)
|
|
349
|
+
for sections in zip(*[iter(todosec1[i]) for i in iprefixes]):
|
|
350
|
+
discovered.tododata.append(
|
|
351
|
+
{
|
|
352
|
+
self._MODELSIDE_INPUTPREFIX0 + k: v
|
|
353
|
+
for k, v in zip(iprefixes, sections)
|
|
354
|
+
}
|
|
355
|
+
)
|
|
303
356
|
|
|
304
357
|
# Detect the number of terms based on the firstprefix
|
|
305
358
|
if hasterms:
|
|
306
359
|
for sections in discovered.tododata:
|
|
307
|
-
act_term = self._actual_term(
|
|
308
|
-
|
|
360
|
+
act_term = self._actual_term(
|
|
361
|
+
sections[
|
|
362
|
+
self._MODELSIDE_INPUTPREFIX0 + discovered.firstprefix
|
|
363
|
+
].rh
|
|
364
|
+
)
|
|
309
365
|
discovered.termscount[act_term] += 1
|
|
310
366
|
|
|
311
367
|
# Look for guesses of output files
|
|
312
368
|
guesses_sec0 = collections.defaultdict(list)
|
|
313
|
-
guess_entry = collections.namedtuple(
|
|
314
|
-
|
|
369
|
+
guess_entry = collections.namedtuple(
|
|
370
|
+
"guess_entry", ("sdir", "prefix", "domain", "suffix", "sec")
|
|
371
|
+
)
|
|
372
|
+
for sec in self.context.sequence.effective_inputs(
|
|
373
|
+
role=self._OUTPUTGUESS_ROLE
|
|
374
|
+
):
|
|
315
375
|
s_lpath = sec.rh.container.localpath()
|
|
316
376
|
s_match = self._o_algo_re.match(self.system.path.basename(s_lpath))
|
|
317
377
|
if s_match:
|
|
318
|
-
guesses_sec0[s_match.group(
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
378
|
+
guesses_sec0[s_match.group("base")].append(
|
|
379
|
+
guess_entry(
|
|
380
|
+
self.system.path.dirname(s_lpath),
|
|
381
|
+
self._o_auto_prefix(
|
|
382
|
+
"grib"
|
|
383
|
+
if s_match.group("grib")
|
|
384
|
+
else self.defaultformat
|
|
385
|
+
),
|
|
386
|
+
s_match.group("fpdom"),
|
|
387
|
+
s_match.group("suffix"),
|
|
388
|
+
sec,
|
|
389
|
+
)
|
|
390
|
+
)
|
|
391
|
+
discovered.anyexpected = (
|
|
392
|
+
discovered.anyexpected or sec.rh.is_expected()
|
|
393
|
+
)
|
|
326
394
|
else:
|
|
327
|
-
logger.warning(
|
|
395
|
+
logger.warning(
|
|
396
|
+
"Improper name for the following output guess < %s >. Ignoring it.",
|
|
397
|
+
s_lpath,
|
|
398
|
+
)
|
|
328
399
|
# Pair them with input file (based on their name)
|
|
329
400
|
for iinput in discovered.tododata:
|
|
330
|
-
isec = iinput[
|
|
401
|
+
isec = iinput[
|
|
402
|
+
self._MODELSIDE_INPUTPREFIX0 + discovered.firstprefix
|
|
403
|
+
]
|
|
331
404
|
discovered.guessdata.append(
|
|
332
|
-
guesses_sec0.pop(
|
|
405
|
+
guesses_sec0.pop(
|
|
406
|
+
self.system.path.basename(isec.rh.container.localpath()),
|
|
407
|
+
(),
|
|
408
|
+
)
|
|
333
409
|
)
|
|
334
410
|
if guesses_sec0:
|
|
335
|
-
logger.warning(
|
|
336
|
-
|
|
411
|
+
logger.warning(
|
|
412
|
+
"Some input data were left unsed: < %s >", guesses_sec0
|
|
413
|
+
)
|
|
414
|
+
logger.info(
|
|
415
|
+
"discovered guessdata are: < %s >", discovered.guessdata
|
|
416
|
+
)
|
|
337
417
|
|
|
338
418
|
return discovered
|
|
339
419
|
|
|
340
420
|
@cached_property
|
|
341
421
|
def object_namelists(self):
|
|
342
422
|
"""The list of object's namelists."""
|
|
343
|
-
namrhs = [
|
|
344
|
-
|
|
345
|
-
|
|
423
|
+
namrhs = [
|
|
424
|
+
isec.rh
|
|
425
|
+
for isec in self.context.sequence.effective_inputs(
|
|
426
|
+
role="ObjectNamelist"
|
|
427
|
+
)
|
|
428
|
+
if isec.rh.resource.realkind == "namelist_fpobject"
|
|
429
|
+
]
|
|
346
430
|
# Update the object's content
|
|
347
431
|
for namrh in namrhs:
|
|
348
432
|
namsave = False
|
|
349
433
|
if namrh.resource.fp_cmodel is not None:
|
|
350
|
-
self._set_nam_macro(
|
|
351
|
-
|
|
434
|
+
self._set_nam_macro(
|
|
435
|
+
namrh.contents,
|
|
436
|
+
namrh.container.localpath(),
|
|
437
|
+
"FP_CMODEL",
|
|
438
|
+
namrh.resource.fp_cmodel,
|
|
439
|
+
)
|
|
352
440
|
namsave = True
|
|
353
441
|
if namrh.resource.fp_lextern is not None:
|
|
354
|
-
self._set_nam_macro(
|
|
355
|
-
|
|
442
|
+
self._set_nam_macro(
|
|
443
|
+
namrh.contents,
|
|
444
|
+
namrh.container.localpath(),
|
|
445
|
+
"FP_LEXTERN",
|
|
446
|
+
namrh.resource.fp_lextern,
|
|
447
|
+
)
|
|
356
448
|
namsave = True
|
|
357
449
|
if namrh.resource.fp_terms is not None:
|
|
358
450
|
if not self.inputs.termscount:
|
|
359
|
-
raise AlgoComponentError(
|
|
451
|
+
raise AlgoComponentError(
|
|
452
|
+
"In this use case, all input data must have a term attribute"
|
|
453
|
+
)
|
|
360
454
|
active_terms = {Time(t) for t in namrh.resource.fp_terms}
|
|
361
455
|
# Generate the list of NFPOSTS
|
|
362
456
|
global_i = 0
|
|
@@ -367,23 +461,34 @@ class FullPosServer(IFSParallel):
|
|
|
367
461
|
global_i += n_term
|
|
368
462
|
# Get the NAMFPC block
|
|
369
463
|
try:
|
|
370
|
-
nfpc = namrh.contents[
|
|
464
|
+
nfpc = namrh.contents["NAMFPC"]
|
|
371
465
|
except KeyError:
|
|
372
|
-
raise AlgoComponentError(
|
|
373
|
-
|
|
466
|
+
raise AlgoComponentError(
|
|
467
|
+
"NAMFPC should be defined in {:s}".format(
|
|
468
|
+
namrh.container.localpath()
|
|
469
|
+
)
|
|
470
|
+
)
|
|
374
471
|
# Sanity check
|
|
375
472
|
for k in nfpc.keys():
|
|
376
|
-
if k.startswith(
|
|
377
|
-
raise AlgoComponentError(
|
|
378
|
-
|
|
473
|
+
if k.startswith("NFPOSTS"):
|
|
474
|
+
raise AlgoComponentError(
|
|
475
|
+
"&NAMFPC NFPOSTS*(*) / entries should not be defined in {:s}".format(
|
|
476
|
+
namrh.container.localpath()
|
|
477
|
+
)
|
|
478
|
+
)
|
|
379
479
|
# Write NFPOSTS to NAMFPC
|
|
380
|
-
nfpc[
|
|
480
|
+
nfpc["NFPOSTS(0)"] = -len(nfposts)
|
|
381
481
|
for i, v in enumerate(nfposts):
|
|
382
|
-
nfpc[
|
|
383
|
-
logger.info(
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
482
|
+
nfpc["NFPOSTS({:d})".format(i + 1)] = -v
|
|
483
|
+
logger.info(
|
|
484
|
+
"The NAMFPC namelist in %s was updated.",
|
|
485
|
+
namrh.container.localpath(),
|
|
486
|
+
)
|
|
487
|
+
logger.debug(
|
|
488
|
+
"The updated NAMFPC namelist in %s is:\n%s",
|
|
489
|
+
namrh.container.localpath(),
|
|
490
|
+
nfpc,
|
|
491
|
+
)
|
|
387
492
|
namsave = True
|
|
388
493
|
if namsave:
|
|
389
494
|
namrh.save()
|
|
@@ -393,53 +498,81 @@ class FullPosServer(IFSParallel):
|
|
|
393
498
|
def xxtmapping(self):
|
|
394
499
|
"""A handy dictionary about selection namelists."""
|
|
395
500
|
namxxrh = collections.defaultdict(dict)
|
|
396
|
-
for isec in self.context.sequence.effective_inputs(
|
|
397
|
-
|
|
501
|
+
for isec in self.context.sequence.effective_inputs(
|
|
502
|
+
role="FullPosSelection", kind="namselect"
|
|
503
|
+
):
|
|
398
504
|
dpath = self.system.path.dirname(isec.rh.container.localpath())
|
|
399
505
|
namxxrh[dpath][isec.rh.resource.term] = isec.rh
|
|
400
506
|
if namxxrh and not self.inputs.termscount:
|
|
401
|
-
raise AlgoComponentError(
|
|
507
|
+
raise AlgoComponentError(
|
|
508
|
+
"In this use case, all input data must have a term attribute"
|
|
509
|
+
)
|
|
402
510
|
return namxxrh
|
|
403
511
|
|
|
404
512
|
@cached_property
|
|
405
513
|
def _i_fmt(self):
|
|
406
514
|
"""The input files format (as expected by the c903)."""
|
|
407
|
-
return (
|
|
408
|
-
|
|
515
|
+
return (
|
|
516
|
+
"{:s}"
|
|
517
|
+
+ "{:s}+".format(self.xpname)
|
|
518
|
+
+ "{:0"
|
|
519
|
+
+ str(self.inputs.actual_suffixlen())
|
|
520
|
+
+ "d}"
|
|
521
|
+
)
|
|
409
522
|
|
|
410
523
|
@cached_property
|
|
411
524
|
def _o_raw_fmt(self):
|
|
412
525
|
"""The output files format (as imposed by the c903)."""
|
|
413
|
-
return (
|
|
414
|
-
|
|
526
|
+
return (
|
|
527
|
+
"{:s}"
|
|
528
|
+
+ "{:s}".format(self.xpname)
|
|
529
|
+
+ "{:s}+"
|
|
530
|
+
+ "{:0"
|
|
531
|
+
+ str(
|
|
532
|
+
self.inputs.actual_suffixlen(self._MODELSIDE_OUT_SUFFIXLEN_MIN)
|
|
533
|
+
)
|
|
534
|
+
+ "d}{:s}"
|
|
535
|
+
)
|
|
415
536
|
|
|
416
537
|
@cached_property
|
|
417
538
|
def _o_re_fmt(self):
|
|
418
539
|
"""The output files regex (as imposed by the c903)."""
|
|
419
|
-
return (
|
|
420
|
-
|
|
421
|
-
|
|
540
|
+
return (
|
|
541
|
+
"^{:s}"
|
|
542
|
+
+ "{:s}".format(self.xpname)
|
|
543
|
+
+ r"(?P<fpdom>\w+)\+"
|
|
544
|
+
+ "{:0"
|
|
545
|
+
+ str(
|
|
546
|
+
self.inputs.actual_suffixlen(self._MODELSIDE_OUT_SUFFIXLEN_MIN)
|
|
547
|
+
)
|
|
548
|
+
+ r"d}(?P<suffix>(?:\.sfx)?)$"
|
|
549
|
+
)
|
|
422
550
|
|
|
423
551
|
@cached_property
|
|
424
552
|
def _o_init_re_fmt(self):
|
|
425
553
|
"""The output files regex (as imposed by the c903)."""
|
|
426
|
-
return (
|
|
427
|
-
|
|
554
|
+
return (
|
|
555
|
+
"^{:s}"
|
|
556
|
+
+ "{:s}".format(self.xpname)
|
|
557
|
+
+ r"(?P<fpdom>\w+){:s}(?P<suffix>(?:\.sfx)?)$"
|
|
558
|
+
)
|
|
428
559
|
|
|
429
560
|
@cached_property
|
|
430
561
|
def _o_algo_re(self):
|
|
431
562
|
"""The regex for any output (as imposed by our AlgoComponent)."""
|
|
432
|
-
return re.compile(
|
|
563
|
+
return re.compile(
|
|
564
|
+
r"(?P<base>.+)\.(?P<fpdom>\w+)(?P<suffix>(?:\.sfx)?)(?P<grib>(?:\.grib)?)\.out$"
|
|
565
|
+
)
|
|
433
566
|
|
|
434
567
|
@cached_property
|
|
435
568
|
def _o_suffix(self):
|
|
436
569
|
"""The FAs output suffix (as imposed by our AlgoComponent)."""
|
|
437
|
-
return
|
|
570
|
+
return ".{:s}{:s}.out"
|
|
438
571
|
|
|
439
572
|
@cached_property
|
|
440
573
|
def _o_grb_suffix(self):
|
|
441
574
|
"""The GRIBs output suffix (as imposed by our AlgoComponent)."""
|
|
442
|
-
return
|
|
575
|
+
return ".{:s}{:s}.grib.out"
|
|
443
576
|
|
|
444
577
|
def _o_auto_prefix(self, fmt):
|
|
445
578
|
"""Return the appropriate output files prefix (as imposed by the c903)."""
|
|
@@ -462,80 +595,132 @@ class FullPosServer(IFSParallel):
|
|
|
462
595
|
outputs_mapping[re.compile(re_default)] = what_default
|
|
463
596
|
# GRIB files
|
|
464
597
|
re_grib = out_re.format(self._MODELSIDE_OUTPUTPREFIX_GRIB, i)
|
|
465
|
-
what_grib = (out_fname + self._o_grb_suffix,
|
|
598
|
+
what_grib = (out_fname + self._o_grb_suffix, "grib")
|
|
466
599
|
outputs_mapping[re.compile(re_grib)] = what_grib
|
|
467
|
-
logger.info(
|
|
468
|
-
|
|
600
|
+
logger.info(
|
|
601
|
+
"Output %s mapped as %s. Output %s mapped as %s.",
|
|
602
|
+
re_default,
|
|
603
|
+
what_default[0],
|
|
604
|
+
re_grib,
|
|
605
|
+
what_grib[0],
|
|
606
|
+
)
|
|
469
607
|
|
|
470
608
|
def _link_input(self, iprefix, irh, i, inputs_mapping, outputs_mapping):
|
|
471
609
|
"""Link an input file and update the mappings dictionaries."""
|
|
472
610
|
sourcepath = irh.container.localpath()
|
|
473
611
|
inputs_mapping[sourcepath] = self._i_fmt.format(iprefix, i)
|
|
474
|
-
self.system.cp(
|
|
475
|
-
|
|
612
|
+
self.system.cp(
|
|
613
|
+
sourcepath,
|
|
614
|
+
inputs_mapping[sourcepath],
|
|
615
|
+
intent="in",
|
|
616
|
+
fmt=irh.container.actualfmt,
|
|
617
|
+
)
|
|
618
|
+
logger.info("%s copied as %s.", sourcepath, inputs_mapping[sourcepath])
|
|
476
619
|
if iprefix == self._MODELSIDE_INPUTPREFIX0 + self.inputs.firstprefix:
|
|
477
|
-
self._add_output_mapping(
|
|
478
|
-
|
|
620
|
+
self._add_output_mapping(
|
|
621
|
+
outputs_mapping,
|
|
622
|
+
i,
|
|
623
|
+
self._o_re_fmt,
|
|
624
|
+
self.system.path.basename(sourcepath),
|
|
625
|
+
)
|
|
479
626
|
if self.append_domain:
|
|
480
627
|
outputpath = self._o_raw_fmt.format(
|
|
481
|
-
self._o_auto_prefix(irh.container.actualfmt),
|
|
628
|
+
self._o_auto_prefix(irh.container.actualfmt),
|
|
629
|
+
self.append_domain,
|
|
630
|
+
i,
|
|
631
|
+
"",
|
|
482
632
|
)
|
|
483
633
|
|
|
484
634
|
if self.outdirectories:
|
|
485
|
-
todo = [
|
|
635
|
+
todo = [
|
|
636
|
+
self.system.path.join(d, outputpath)
|
|
637
|
+
for d in self.outdirectories
|
|
638
|
+
]
|
|
486
639
|
else:
|
|
487
|
-
todo = [
|
|
640
|
+
todo = [
|
|
641
|
+
outputpath,
|
|
642
|
+
]
|
|
488
643
|
for a_outputpath in todo:
|
|
489
|
-
self.system.cp(
|
|
490
|
-
|
|
644
|
+
self.system.cp(
|
|
645
|
+
sourcepath,
|
|
646
|
+
a_outputpath,
|
|
647
|
+
intent="inout",
|
|
648
|
+
fmt=irh.container.actualfmt,
|
|
649
|
+
)
|
|
650
|
+
logger.info(
|
|
651
|
+
"output file prepared: %s copied (rw) to %s.",
|
|
652
|
+
sourcepath,
|
|
653
|
+
a_outputpath,
|
|
654
|
+
)
|
|
491
655
|
|
|
492
656
|
def _move_output_guess(self, iguess, i):
|
|
493
657
|
"""Move the output file guesses to their final location."""
|
|
494
658
|
sourcepath = iguess.sec.rh.container.localpath()
|
|
495
659
|
destpath = self.system.path.join(
|
|
496
660
|
iguess.sdir,
|
|
497
|
-
self._o_raw_fmt.format(
|
|
661
|
+
self._o_raw_fmt.format(
|
|
662
|
+
iguess.prefix, iguess.domain, i, iguess.suffix
|
|
663
|
+
),
|
|
664
|
+
)
|
|
665
|
+
self.system.mv(
|
|
666
|
+
sourcepath, destpath, fmt=iguess.sec.rh.container.actualfmt
|
|
498
667
|
)
|
|
499
|
-
|
|
500
|
-
logger.info('output guess %s was moved to %s.', sourcepath, destpath)
|
|
668
|
+
logger.info("output guess %s was moved to %s.", sourcepath, destpath)
|
|
501
669
|
|
|
502
670
|
def _link_xxt(self, todorh, i):
|
|
503
671
|
"""If necessary, link in the appropriate xxtNNNNNNMM file."""
|
|
504
672
|
for sdir, tdict in self.xxtmapping.items():
|
|
505
673
|
xxtrh = tdict.get(self._actual_term(todorh), None)
|
|
506
674
|
if xxtrh is not None:
|
|
507
|
-
xxtsource = self.system.path.relpath(
|
|
508
|
-
|
|
675
|
+
xxtsource = self.system.path.relpath(
|
|
676
|
+
xxtrh.container.abspath, sdir
|
|
677
|
+
)
|
|
509
678
|
# The file is expected to follow the xxtDDDDHHMM syntax where DDDD
|
|
510
679
|
# is the number of days
|
|
511
680
|
days_hours = (i // 24) * 100 + i % 24
|
|
512
|
-
xxttarget =
|
|
681
|
+
xxttarget = "xxt{:06d}00".format(days_hours)
|
|
513
682
|
xxttarget = self.system.path.join(sdir, xxttarget)
|
|
514
683
|
self.system.symlink(xxtsource, xxttarget)
|
|
515
|
-
logger.info(
|
|
684
|
+
logger.info("XXT %s linked in as %s.", xxtsource, xxttarget)
|
|
516
685
|
|
|
517
686
|
def _init_poll_and_move(self, outputs_mapping):
|
|
518
687
|
"""Deal with the PF*INIT file."""
|
|
519
688
|
sh = self.system
|
|
520
|
-
candidates = self.system.glob(
|
|
689
|
+
candidates = self.system.glob(
|
|
690
|
+
"{:s}{:s}*INIT".format(self._MODELSIDE_OUTPUTPREFIX, self.xpname)
|
|
691
|
+
)
|
|
521
692
|
outputnames = list()
|
|
522
693
|
for thisdata in candidates:
|
|
523
694
|
mappeddata = None
|
|
524
695
|
for out_re, data in outputs_mapping.items():
|
|
525
696
|
m_re = out_re.match(thisdata)
|
|
526
697
|
if m_re:
|
|
527
|
-
mappeddata = (
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
698
|
+
mappeddata = (
|
|
699
|
+
sh.path.join(
|
|
700
|
+
sh.path.dirname(thisdata),
|
|
701
|
+
data[0].format(
|
|
702
|
+
m_re.group("fpdom"), m_re.group("suffix")
|
|
703
|
+
),
|
|
704
|
+
),
|
|
705
|
+
data[1],
|
|
706
|
+
)
|
|
531
707
|
break
|
|
532
708
|
if mappeddata is None:
|
|
533
|
-
raise AlgoComponentError(
|
|
709
|
+
raise AlgoComponentError(
|
|
710
|
+
"The mapping failed for {:s}.".format(thisdata)
|
|
711
|
+
)
|
|
534
712
|
# Already dealt with ?
|
|
535
713
|
if not self.system.path.exists(mappeddata[0]):
|
|
536
|
-
logger.info(
|
|
714
|
+
logger.info(
|
|
715
|
+
"Linking <%s> to <%s> (fmt=%s).",
|
|
716
|
+
thisdata,
|
|
717
|
+
mappeddata[0],
|
|
718
|
+
mappeddata[1],
|
|
719
|
+
)
|
|
537
720
|
outputnames.append(mappeddata[0])
|
|
538
|
-
self.system.cp(
|
|
721
|
+
self.system.cp(
|
|
722
|
+
thisdata, mappeddata[0], intent="in", fmt=mappeddata[1]
|
|
723
|
+
)
|
|
539
724
|
return outputnames
|
|
540
725
|
|
|
541
726
|
def _poll_and_move(self, outputs_mapping):
|
|
@@ -548,26 +733,44 @@ class FullPosServer(IFSParallel):
|
|
|
548
733
|
for out_re, data in outputs_mapping.items():
|
|
549
734
|
m_re = out_re.match(sh.path.basename(thisdata))
|
|
550
735
|
if m_re:
|
|
551
|
-
mappeddata = (
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
736
|
+
mappeddata = (
|
|
737
|
+
sh.path.join(
|
|
738
|
+
sh.path.dirname(thisdata),
|
|
739
|
+
data[0].format(
|
|
740
|
+
m_re.group("fpdom"), m_re.group("suffix")
|
|
741
|
+
),
|
|
742
|
+
),
|
|
743
|
+
data[1],
|
|
744
|
+
)
|
|
555
745
|
break
|
|
556
746
|
if mappeddata is None:
|
|
557
|
-
raise AlgoComponentError(
|
|
558
|
-
|
|
747
|
+
raise AlgoComponentError(
|
|
748
|
+
"The mapping failed for {:s}.".format(thisdata)
|
|
749
|
+
)
|
|
750
|
+
logger.info(
|
|
751
|
+
"Linking <%s> to <%s> (fmt=%s).",
|
|
752
|
+
thisdata,
|
|
753
|
+
mappeddata[0],
|
|
754
|
+
mappeddata[1],
|
|
755
|
+
)
|
|
559
756
|
outputnames.append(mappeddata[0])
|
|
560
|
-
self.system.cp(
|
|
757
|
+
self.system.cp(
|
|
758
|
+
thisdata, mappeddata[0], intent="in", fmt=mappeddata[1]
|
|
759
|
+
)
|
|
561
760
|
return outputnames
|
|
562
761
|
|
|
563
762
|
def _deal_with_promises(self, outputs_mapping, pollingcb):
|
|
564
763
|
if self.promises:
|
|
565
764
|
seen = pollingcb(outputs_mapping)
|
|
566
765
|
for afile in seen:
|
|
567
|
-
candidates = [
|
|
568
|
-
|
|
766
|
+
candidates = [
|
|
767
|
+
x
|
|
768
|
+
for x in self.promises
|
|
769
|
+
if x.rh.container.abspath
|
|
770
|
+
== self.system.path.abspath(afile)
|
|
771
|
+
]
|
|
569
772
|
if candidates:
|
|
570
|
-
logger.info(
|
|
773
|
+
logger.info("The output data is promised <%s>", afile)
|
|
571
774
|
bingo = candidates.pop()
|
|
572
775
|
bingo.put(incache=True)
|
|
573
776
|
|
|
@@ -576,129 +779,214 @@ class FullPosServer(IFSParallel):
|
|
|
576
779
|
super().prepare(rh, opts)
|
|
577
780
|
|
|
578
781
|
if self.object_namelists:
|
|
579
|
-
self.system.subtitle(
|
|
782
|
+
self.system.subtitle("Object Namelists customisation")
|
|
580
783
|
for o_nam in self.object_namelists:
|
|
581
784
|
# a/c cy44: &NAMFPIOS NFPDIGITS=__SUFFIXLEN__, /
|
|
582
|
-
self._set_nam_macro(
|
|
583
|
-
|
|
785
|
+
self._set_nam_macro(
|
|
786
|
+
o_nam.contents,
|
|
787
|
+
o_nam.container.localpath(),
|
|
788
|
+
"SUFFIXLEN",
|
|
789
|
+
self.inputs.actual_suffixlen(
|
|
790
|
+
self._MODELSIDE_OUT_SUFFIXLEN_MIN
|
|
791
|
+
),
|
|
792
|
+
)
|
|
584
793
|
if o_nam.contents.dumps_needs_update:
|
|
585
|
-
logger.info(
|
|
794
|
+
logger.info(
|
|
795
|
+
"Rewritting the %s namelists file.",
|
|
796
|
+
o_nam.container.actualpath(),
|
|
797
|
+
)
|
|
586
798
|
o_nam.save()
|
|
587
799
|
|
|
588
|
-
self.system.subtitle(
|
|
800
|
+
self.system.subtitle("Dealing with various input files")
|
|
589
801
|
|
|
590
802
|
# Sanity check over climfiles and geometries
|
|
591
|
-
input_geo = {
|
|
592
|
-
|
|
803
|
+
input_geo = {
|
|
804
|
+
sec.rh.resource.geometry
|
|
805
|
+
for sdict in self.inputs.tododata
|
|
806
|
+
for sec in sdict.values()
|
|
807
|
+
}
|
|
593
808
|
if len(input_geo) == 0:
|
|
594
|
-
raise AlgoComponentError(
|
|
809
|
+
raise AlgoComponentError("No input data are provided, ...")
|
|
595
810
|
elif len(input_geo) > 1:
|
|
596
|
-
raise AlgoComponentError(
|
|
811
|
+
raise AlgoComponentError(
|
|
812
|
+
"Multiple geometries are not allowed for input data."
|
|
813
|
+
)
|
|
597
814
|
else:
|
|
598
815
|
input_geo = input_geo.pop()
|
|
599
816
|
|
|
600
|
-
input_climgeo = {
|
|
601
|
-
|
|
602
|
-
|
|
817
|
+
input_climgeo = {
|
|
818
|
+
x.rh.resource.geometry
|
|
819
|
+
for x in self.context.sequence.effective_inputs(
|
|
820
|
+
role=("InputClim", "InitialClim")
|
|
821
|
+
)
|
|
822
|
+
}
|
|
603
823
|
if len(input_climgeo) == 0:
|
|
604
|
-
logger.info(
|
|
824
|
+
logger.info("No input clim provided. Going on without it...")
|
|
605
825
|
elif len(input_climgeo) > 1:
|
|
606
|
-
raise AlgoComponentError(
|
|
826
|
+
raise AlgoComponentError(
|
|
827
|
+
"Multiple geometries are not allowed for input climatology."
|
|
828
|
+
)
|
|
607
829
|
else:
|
|
608
830
|
if input_climgeo.pop() != input_geo:
|
|
609
|
-
raise AlgoComponentError(
|
|
831
|
+
raise AlgoComponentError(
|
|
832
|
+
"The input data and input climatology geometries does not match."
|
|
833
|
+
)
|
|
610
834
|
|
|
611
835
|
# Initial Condition geometry sanity check
|
|
612
|
-
if self.inputs.inidata and any(
|
|
613
|
-
|
|
614
|
-
|
|
836
|
+
if self.inputs.inidata and any(
|
|
837
|
+
[
|
|
838
|
+
sec.rh.resource.geometry != input_geo
|
|
839
|
+
for sec in self.inputs.inidata.values()
|
|
840
|
+
]
|
|
841
|
+
):
|
|
842
|
+
raise AlgoComponentError(
|
|
843
|
+
"The Initial Condition geometry differs from other input data."
|
|
844
|
+
)
|
|
615
845
|
|
|
616
846
|
# Sanity check on target climatology files
|
|
617
|
-
target_climgeos = {
|
|
618
|
-
|
|
847
|
+
target_climgeos = {
|
|
848
|
+
x.rh.resource.geometry
|
|
849
|
+
for x in self.context.sequence.effective_inputs(role="TargetClim")
|
|
850
|
+
}
|
|
619
851
|
if len(target_climgeos) == 0:
|
|
620
|
-
logger.info(
|
|
852
|
+
logger.info("No target clim are provided. Going on without it...")
|
|
621
853
|
|
|
622
854
|
# Sanity check on selection namelists
|
|
623
855
|
if self.xxtmapping:
|
|
624
856
|
for tdict in self.xxtmapping.values():
|
|
625
|
-
if
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
857
|
+
if {
|
|
858
|
+
self._actual_term(sec.rh)
|
|
859
|
+
for sdict in self.inputs.tododata
|
|
860
|
+
for sec in sdict.values()
|
|
861
|
+
} < set(tdict.keys()):
|
|
862
|
+
raise AlgoComponentError(
|
|
863
|
+
"The list of terms between input data and selection namelists differs"
|
|
864
|
+
)
|
|
629
865
|
else:
|
|
630
866
|
logger.info("No selection namelists detected. That's fine")
|
|
631
867
|
|
|
632
868
|
# Link in the initial condition file (if necessary)
|
|
633
869
|
for iprefix, isec in self.inputs.inidata.items():
|
|
634
|
-
i_init =
|
|
870
|
+
i_init = "{:s}{:s}INIT".format(iprefix, self.xpname)
|
|
635
871
|
if isec.rh.container.basename != i_init:
|
|
636
|
-
self.system.cp(
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
872
|
+
self.system.cp(
|
|
873
|
+
isec.rh.container.localpath(),
|
|
874
|
+
i_init,
|
|
875
|
+
intent="in",
|
|
876
|
+
fmt=isec.rh.container.actualfmt,
|
|
877
|
+
)
|
|
878
|
+
logger.info(
|
|
879
|
+
"Initial condition file %s copied as %s.",
|
|
880
|
+
isec.rh.container.localpath(),
|
|
881
|
+
i_init,
|
|
882
|
+
)
|
|
640
883
|
|
|
641
884
|
def find_namelists(self, opts=None):
|
|
642
885
|
"""Find any namelists candidates in actual context inputs."""
|
|
643
|
-
return [
|
|
644
|
-
|
|
645
|
-
|
|
886
|
+
return [
|
|
887
|
+
x.rh
|
|
888
|
+
for x in self.context.sequence.effective_inputs(
|
|
889
|
+
role="Namelist", kind="namelist"
|
|
890
|
+
)
|
|
891
|
+
]
|
|
646
892
|
|
|
647
893
|
def prepare_namelist_delta(self, rh, namcontents, namlocal):
|
|
648
894
|
super().prepare_namelist_delta(rh, namcontents, namlocal)
|
|
649
895
|
# With cy43: &NAMCT0 CSCRIPT_PPSERVER=__SERVERSYNC_SCRIPT__, /
|
|
650
896
|
if self.inputs.anyexpected:
|
|
651
|
-
self._set_nam_macro(
|
|
652
|
-
|
|
897
|
+
self._set_nam_macro(
|
|
898
|
+
namcontents,
|
|
899
|
+
namlocal,
|
|
900
|
+
"SERVERSYNC_SCRIPT",
|
|
901
|
+
self.system.path.join(".", self.serversync_medium),
|
|
902
|
+
)
|
|
653
903
|
else:
|
|
654
904
|
# Do not harass the filesystem...
|
|
655
|
-
self._set_nam_macro(
|
|
905
|
+
self._set_nam_macro(
|
|
906
|
+
namcontents, namlocal, "SERVERSYNC_SCRIPT", " "
|
|
907
|
+
)
|
|
656
908
|
# With cy43: &NAMCT0 CFPNCF=__IOPOLL_WHITNESSFILE__, /
|
|
657
|
-
self._set_nam_macro(
|
|
909
|
+
self._set_nam_macro(
|
|
910
|
+
namcontents,
|
|
911
|
+
namlocal,
|
|
912
|
+
"IOPOLL_WHITNESSFILE",
|
|
913
|
+
self._MODELSIDE_TERMFILE,
|
|
914
|
+
)
|
|
658
915
|
# With cy43: No matching namelist key
|
|
659
916
|
# a/c cy44: &NAMFPIOS NFPDIGITS=__SUFFIXLEN__, /
|
|
660
|
-
self._set_nam_macro(
|
|
661
|
-
|
|
917
|
+
self._set_nam_macro(
|
|
918
|
+
namcontents,
|
|
919
|
+
namlocal,
|
|
920
|
+
"SUFFIXLEN",
|
|
921
|
+
self.inputs.actual_suffixlen(self._MODELSIDE_OUT_SUFFIXLEN_MIN),
|
|
922
|
+
)
|
|
662
923
|
# No matching namelist yet
|
|
663
|
-
self._set_nam_macro(
|
|
664
|
-
|
|
924
|
+
self._set_nam_macro(
|
|
925
|
+
namcontents,
|
|
926
|
+
namlocal,
|
|
927
|
+
"INPUT_SUFFIXLEN",
|
|
928
|
+
self.inputs.actual_suffixlen(),
|
|
929
|
+
)
|
|
665
930
|
# With cy43: &NAMCT0 NFRPOS=__INPUTDATALEN__, /
|
|
666
|
-
self._set_nam_macro(
|
|
931
|
+
self._set_nam_macro(
|
|
932
|
+
namcontents, namlocal, "INPUTDATALEN", -len(self.inputs.tododata)
|
|
933
|
+
)
|
|
667
934
|
# Auto generate the list of namelists for the various objects
|
|
668
935
|
if self.object_namelists:
|
|
669
|
-
if
|
|
670
|
-
|
|
671
|
-
|
|
936
|
+
if (
|
|
937
|
+
"NAMFPOBJ" not in namcontents
|
|
938
|
+
or len(namcontents["NAMFPOBJ"]) == 0
|
|
939
|
+
):
|
|
940
|
+
nb_o = NamelistBlock("NAMFPOBJ")
|
|
941
|
+
nb_o["NFPOBJ"] = len(self.object_namelists)
|
|
672
942
|
for i_nam, nam in enumerate(self.object_namelists):
|
|
673
943
|
if nam.resource.fp_conf:
|
|
674
|
-
nb_o[
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
944
|
+
nb_o["NFPCONF({:d})".format(i_nam + 1)] = (
|
|
945
|
+
nam.resource.fp_conf
|
|
946
|
+
)
|
|
947
|
+
nb_o["CNAMELIST({:d})".format(i_nam + 1)] = (
|
|
948
|
+
nam.container.localpath()
|
|
949
|
+
)
|
|
950
|
+
namcontents["NAMFPOBJ"] = nb_o
|
|
951
|
+
logger.info(
|
|
952
|
+
'The following namelist block has been added to "%s":\n%s',
|
|
953
|
+
namlocal,
|
|
954
|
+
nb_o.dumps(),
|
|
955
|
+
)
|
|
679
956
|
else:
|
|
680
|
-
logger.warning(
|
|
681
|
-
|
|
957
|
+
logger.warning(
|
|
958
|
+
'The NAMFPOBJ namelist in "%s" is not empty. Leaving it as it is',
|
|
959
|
+
namlocal,
|
|
960
|
+
)
|
|
682
961
|
# Just in case FP_CMODEL is defined in the main namelist
|
|
683
|
-
if self.outputid is not None and any(
|
|
684
|
-
|
|
685
|
-
|
|
962
|
+
if self.outputid is not None and any(
|
|
963
|
+
["FP_CMODEL" in nam_b.macros() for nam_b in namcontents.values()]
|
|
964
|
+
):
|
|
965
|
+
self._set_nam_macro(
|
|
966
|
+
namcontents, namlocal, "FP_CMODEL", self.outputid
|
|
967
|
+
)
|
|
686
968
|
return True
|
|
687
969
|
|
|
688
970
|
def spawn_pre_dirlisting(self):
|
|
689
971
|
"""Print a directory listing just before run."""
|
|
690
972
|
super().spawn_pre_dirlisting()
|
|
691
973
|
for sdir in self.outdirectories:
|
|
692
|
-
self.system.subtitle(
|
|
693
|
-
|
|
974
|
+
self.system.subtitle(
|
|
975
|
+
"{:s} : {:s} sub-directory listing (pre-execution)".format(
|
|
976
|
+
self.realkind, sdir
|
|
977
|
+
)
|
|
978
|
+
)
|
|
694
979
|
self.system.dir(sdir, output=False, fatal=False)
|
|
695
980
|
|
|
696
981
|
def spawn_hook(self):
|
|
697
982
|
"""Usually a good habit to dump the fort.4 namelist."""
|
|
698
983
|
super().spawn_hook()
|
|
699
984
|
for o_nam in self.object_namelists:
|
|
700
|
-
self.system.subtitle(
|
|
701
|
-
|
|
985
|
+
self.system.subtitle(
|
|
986
|
+
"{:s} : dump namelist <{:s}>".format(
|
|
987
|
+
self.realkind, o_nam.container.localpath()
|
|
988
|
+
)
|
|
989
|
+
)
|
|
702
990
|
self.system.cat(o_nam.container.localpath(), output=False)
|
|
703
991
|
|
|
704
992
|
def execute(self, rh, opts):
|
|
@@ -716,60 +1004,90 @@ class FullPosServer(IFSParallel):
|
|
|
716
1004
|
self.grab(isec)
|
|
717
1005
|
# Fix potential links and output mappings
|
|
718
1006
|
sourcepath = isec.rh.container.basename
|
|
719
|
-
if
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
1007
|
+
if (
|
|
1008
|
+
iprefix
|
|
1009
|
+
== self._MODELSIDE_INPUTPREFIX0 + self.inputs.firstprefix
|
|
1010
|
+
):
|
|
1011
|
+
self._add_output_mapping(
|
|
1012
|
+
outputs_mapping,
|
|
1013
|
+
"INIT",
|
|
1014
|
+
self._o_init_re_fmt,
|
|
1015
|
+
sourcepath,
|
|
1016
|
+
)
|
|
1017
|
+
i_init = "{:s}{:s}INIT".format(iprefix, self.xpname)
|
|
723
1018
|
if isec.rh.container.basename != i_init:
|
|
724
|
-
self.system.cp(
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
1019
|
+
self.system.cp(
|
|
1020
|
+
sourcepath,
|
|
1021
|
+
i_init,
|
|
1022
|
+
intent="in",
|
|
1023
|
+
fmt=isec.rh.container.actualfmt,
|
|
1024
|
+
)
|
|
1025
|
+
logger.info(
|
|
1026
|
+
"Initial condition file %s copied as %s.",
|
|
1027
|
+
isec.rh.container.localpath(),
|
|
1028
|
+
i_init,
|
|
1029
|
+
)
|
|
728
1030
|
else:
|
|
729
1031
|
if self.inputs.tododata:
|
|
730
1032
|
# Just in case the INIT file is transformed
|
|
731
|
-
fakesource =
|
|
732
|
-
|
|
1033
|
+
fakesource = (
|
|
1034
|
+
self._MODELSIDE_INPUTPREFIX0
|
|
1035
|
+
+ self.inputs.firstprefix
|
|
1036
|
+
+ self.xpname
|
|
1037
|
+
+ "INIT"
|
|
1038
|
+
)
|
|
1039
|
+
self._add_output_mapping(
|
|
1040
|
+
outputs_mapping, "INIT", self._o_init_re_fmt, fakesource
|
|
1041
|
+
)
|
|
733
1042
|
|
|
734
1043
|
# Initialise the flying stuff
|
|
735
1044
|
self.flyput = False # Do not use flyput every time...
|
|
736
1045
|
flyprefixes = set()
|
|
737
1046
|
for s in self.promises:
|
|
738
1047
|
lpath = s.rh.container.localpath()
|
|
739
|
-
if lpath.endswith(
|
|
1048
|
+
if lpath.endswith(".grib.out"):
|
|
740
1049
|
flyprefixes.add(self._MODELSIDE_OUTPUTPREFIX_GRIB)
|
|
741
|
-
elif lpath.endswith(
|
|
1050
|
+
elif lpath.endswith(".out"):
|
|
742
1051
|
flyprefixes.add(self._MODELSIDE_OUTPUTPREFIX)
|
|
743
1052
|
self.io_poll_args = tuple(flyprefixes)
|
|
744
1053
|
self.io_poll_kwargs = dict(directories=tuple(set(self.outdirectories)))
|
|
745
1054
|
for directory in set(self.outdirectories):
|
|
746
1055
|
sh.mkdir(directory) # Create possible output directories
|
|
747
|
-
if self.flypoll ==
|
|
1056
|
+
if self.flypoll == "internal":
|
|
748
1057
|
self.io_poll_method = functools.partial(fullpos_server_flypoll, sh)
|
|
749
|
-
self.io_poll_kwargs[
|
|
1058
|
+
self.io_poll_kwargs["termfile"] = sh.path.basename(
|
|
1059
|
+
self._MODELSIDE_TERMFILE
|
|
1060
|
+
)
|
|
750
1061
|
self.flymapping = True
|
|
751
1062
|
self._flyput_mapping_d = outputs_mapping
|
|
752
1063
|
|
|
753
1064
|
# Deal with XXT files
|
|
754
1065
|
if self.xxtmapping:
|
|
755
1066
|
for i, istuff in enumerate(self.inputs.tododata):
|
|
756
|
-
self._link_xxt(
|
|
757
|
-
|
|
1067
|
+
self._link_xxt(
|
|
1068
|
+
istuff[
|
|
1069
|
+
self._MODELSIDE_INPUTPREFIX0 + self.inputs.firstprefix
|
|
1070
|
+
].rh,
|
|
1071
|
+
i,
|
|
1072
|
+
)
|
|
758
1073
|
|
|
759
1074
|
if self.inputs.anyexpected:
|
|
760
1075
|
# Some server sync here...
|
|
761
1076
|
self.server_run = True
|
|
762
|
-
self.system.subtitle(
|
|
1077
|
+
self.system.subtitle("Starting computation with server_run=T")
|
|
763
1078
|
|
|
764
1079
|
# Process the data in chronological order ?
|
|
765
|
-
ordered_processing =
|
|
766
|
-
|
|
767
|
-
|
|
1080
|
+
ordered_processing = self.xxtmapping or any(
|
|
1081
|
+
[
|
|
1082
|
+
o_rh.resource.fp_terms is not None
|
|
1083
|
+
for o_rh in self.object_namelists
|
|
1084
|
+
]
|
|
1085
|
+
)
|
|
768
1086
|
if ordered_processing:
|
|
769
|
-
logger.info(
|
|
1087
|
+
logger.info("Input data will be processed chronologicaly.")
|
|
770
1088
|
|
|
771
1089
|
# IO poll settings
|
|
772
|
-
self.io_poll_kwargs[
|
|
1090
|
+
self.io_poll_kwargs["nthreads"] = self.maxpollingthreads
|
|
773
1091
|
|
|
774
1092
|
# Is there already an Initial Condition file ?
|
|
775
1093
|
# If so, start the binary...
|
|
@@ -779,7 +1097,9 @@ class FullPosServer(IFSParallel):
|
|
|
779
1097
|
if not self.server_alive():
|
|
780
1098
|
logger.error("Server initialisation failed.")
|
|
781
1099
|
return
|
|
782
|
-
self._deal_with_promises(
|
|
1100
|
+
self._deal_with_promises(
|
|
1101
|
+
outputs_mapping, self._init_poll_and_move
|
|
1102
|
+
)
|
|
783
1103
|
|
|
784
1104
|
# Setup the InputMonitor
|
|
785
1105
|
all_entries = set()
|
|
@@ -787,14 +1107,23 @@ class FullPosServer(IFSParallel):
|
|
|
787
1107
|
cur_term = None
|
|
788
1108
|
cur_term_gangs = set()
|
|
789
1109
|
prev_term_gangs = set()
|
|
790
|
-
for istuff, iguesses in zip(
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
1110
|
+
for istuff, iguesses in zip(
|
|
1111
|
+
self.inputs.tododata, self.inputs.guessdata
|
|
1112
|
+
):
|
|
1113
|
+
iinputs = {
|
|
1114
|
+
_lmonitor.InputMonitorEntry(s) for s in istuff.values()
|
|
1115
|
+
}
|
|
1116
|
+
iinputs |= {
|
|
1117
|
+
_lmonitor.InputMonitorEntry(g.sec) for g in iguesses
|
|
1118
|
+
}
|
|
1119
|
+
iterm = self._actual_term(
|
|
1120
|
+
istuff[
|
|
1121
|
+
self._MODELSIDE_INPUTPREFIX0 + self.inputs.firstprefix
|
|
1122
|
+
].rh
|
|
1123
|
+
)
|
|
795
1124
|
all_entries.update(iinputs)
|
|
796
1125
|
bgang = _lmonitor.BasicGang()
|
|
797
|
-
bgang.add_member(*
|
|
1126
|
+
bgang.add_member(*iinputs)
|
|
798
1127
|
igang = _lmonitor.MetaGang()
|
|
799
1128
|
igang.info = (istuff, iguesses, iterm)
|
|
800
1129
|
igang.add_member(bgang)
|
|
@@ -806,13 +1135,16 @@ class FullPosServer(IFSParallel):
|
|
|
806
1135
|
cur_term_gangs = set()
|
|
807
1136
|
if prev_term_gangs:
|
|
808
1137
|
# Wait for the gangs of the previous terms
|
|
809
|
-
igang.add_member(*
|
|
1138
|
+
igang.add_member(*prev_term_gangs)
|
|
810
1139
|
# Save things up for the next time
|
|
811
1140
|
cur_term_gangs.add(igang)
|
|
812
1141
|
cur_term = iterm
|
|
813
1142
|
metagang.add_member(igang)
|
|
814
|
-
bm = _lmonitor.ManualInputMonitor(
|
|
815
|
-
|
|
1143
|
+
bm = _lmonitor.ManualInputMonitor(
|
|
1144
|
+
self.context,
|
|
1145
|
+
all_entries,
|
|
1146
|
+
caching_freq=self.refreshtime,
|
|
1147
|
+
)
|
|
816
1148
|
|
|
817
1149
|
# Start the InputMonitor
|
|
818
1150
|
tmout = False
|
|
@@ -820,52 +1152,76 @@ class FullPosServer(IFSParallel):
|
|
|
820
1152
|
server_stopped = False
|
|
821
1153
|
with bm:
|
|
822
1154
|
while not bm.all_done or len(bm.available) > 0:
|
|
823
|
-
|
|
824
1155
|
# Fetch available inputs and sort them
|
|
825
1156
|
ibatch = list()
|
|
826
1157
|
while metagang.has_collectable():
|
|
827
1158
|
thegang = metagang.pop_collectable()
|
|
828
1159
|
ibatch.append(thegang.info)
|
|
829
|
-
ibatch.sort(
|
|
1160
|
+
ibatch.sort(
|
|
1161
|
+
key=lambda item: item[2]
|
|
1162
|
+
) # Sort according to the term
|
|
830
1163
|
|
|
831
1164
|
# Deal with the various available inputs
|
|
832
|
-
for
|
|
833
|
-
sh.highlight(
|
|
834
|
-
|
|
1165
|
+
for istuff, iguesses, iterm in ibatch:
|
|
1166
|
+
sh.highlight(
|
|
1167
|
+
"The Fullpos Server is triggered (step={:d})...".format(
|
|
1168
|
+
current_i
|
|
1169
|
+
)
|
|
1170
|
+
)
|
|
835
1171
|
|
|
836
1172
|
# Link for the init file (if needed)
|
|
837
1173
|
if current_i == 0 and not self.inputs.inidata:
|
|
838
1174
|
for iprefix, isec in istuff.items():
|
|
839
|
-
i_init =
|
|
1175
|
+
i_init = "{:s}{:s}INIT".format(
|
|
1176
|
+
iprefix, self.xpname
|
|
1177
|
+
)
|
|
840
1178
|
if not sh.path.exists(i_init):
|
|
841
|
-
sh.cp(
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
1179
|
+
sh.cp(
|
|
1180
|
+
isec.rh.container.localpath(),
|
|
1181
|
+
i_init,
|
|
1182
|
+
intent="in",
|
|
1183
|
+
fmt=isec.rh.container.actualfmt,
|
|
1184
|
+
)
|
|
1185
|
+
logger.info(
|
|
1186
|
+
"%s copied as %s. For initialisation purposes only.",
|
|
1187
|
+
isec.rh.container.localpath(),
|
|
1188
|
+
i_init,
|
|
1189
|
+
)
|
|
845
1190
|
super().execute(rh, opts)
|
|
846
1191
|
# Did the server stopped ?
|
|
847
1192
|
if not self.server_alive():
|
|
848
1193
|
logger.error("Server initialisation failed.")
|
|
849
1194
|
return
|
|
850
|
-
self._deal_with_promises(
|
|
1195
|
+
self._deal_with_promises(
|
|
1196
|
+
outputs_mapping, self._init_poll_and_move
|
|
1197
|
+
)
|
|
851
1198
|
|
|
852
1199
|
# Link input files
|
|
853
1200
|
for iprefix, isec in istuff.items():
|
|
854
|
-
self._link_input(
|
|
855
|
-
|
|
1201
|
+
self._link_input(
|
|
1202
|
+
iprefix,
|
|
1203
|
+
isec.rh,
|
|
1204
|
+
current_i,
|
|
1205
|
+
inputs_mapping,
|
|
1206
|
+
outputs_mapping,
|
|
1207
|
+
)
|
|
856
1208
|
for iguess in iguesses:
|
|
857
1209
|
self._move_output_guess(iguess, current_i)
|
|
858
1210
|
|
|
859
1211
|
# Let's go...
|
|
860
1212
|
super().execute(rh, opts)
|
|
861
|
-
self._deal_with_promises(
|
|
1213
|
+
self._deal_with_promises(
|
|
1214
|
+
outputs_mapping, self._poll_and_move
|
|
1215
|
+
)
|
|
862
1216
|
current_i += 1
|
|
863
1217
|
|
|
864
1218
|
# Did the server stopped ?
|
|
865
1219
|
if not self.server_alive():
|
|
866
1220
|
server_stopped = True
|
|
867
1221
|
if not bm.all_done:
|
|
868
|
-
logger.error(
|
|
1222
|
+
logger.error(
|
|
1223
|
+
"The server stopped but everything wasn't processed..."
|
|
1224
|
+
)
|
|
869
1225
|
break
|
|
870
1226
|
|
|
871
1227
|
if server_stopped:
|
|
@@ -880,12 +1236,18 @@ class FullPosServer(IFSParallel):
|
|
|
880
1236
|
time.sleep(1)
|
|
881
1237
|
bm.health_check(interval=30)
|
|
882
1238
|
|
|
883
|
-
for failed_file in [
|
|
884
|
-
|
|
885
|
-
|
|
1239
|
+
for failed_file in [
|
|
1240
|
+
e.section.rh.container.localpath() for e in bm.failed.values()
|
|
1241
|
+
]:
|
|
1242
|
+
logger.error(
|
|
1243
|
+
"We were unable to fetch the following file: %s",
|
|
1244
|
+
failed_file,
|
|
1245
|
+
)
|
|
886
1246
|
if self.fatal:
|
|
887
|
-
self.delayed_exception_add(
|
|
888
|
-
|
|
1247
|
+
self.delayed_exception_add(
|
|
1248
|
+
IOError("Unable to fetch {:s}".format(failed_file)),
|
|
1249
|
+
traceback=False,
|
|
1250
|
+
)
|
|
889
1251
|
|
|
890
1252
|
if tmout:
|
|
891
1253
|
raise OSError("The waiting loop timed out")
|
|
@@ -893,23 +1255,33 @@ class FullPosServer(IFSParallel):
|
|
|
893
1255
|
else:
|
|
894
1256
|
# Direct Run !
|
|
895
1257
|
self.server_run = False
|
|
896
|
-
self.system.subtitle(
|
|
1258
|
+
self.system.subtitle("Starting computation with server_run=F")
|
|
897
1259
|
|
|
898
1260
|
# Link for the inifile (if needed)
|
|
899
1261
|
if not self.inputs.inidata:
|
|
900
1262
|
for iprefix, isec in self.inputs.tododata[0].items():
|
|
901
|
-
i_init =
|
|
1263
|
+
i_init = "{:s}{:s}INIT".format(iprefix, self.xpname)
|
|
902
1264
|
if not sh.path.exists(i_init):
|
|
903
|
-
sh.cp(
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
1265
|
+
sh.cp(
|
|
1266
|
+
isec.rh.container.localpath(),
|
|
1267
|
+
i_init,
|
|
1268
|
+
intent="in",
|
|
1269
|
+
fmt=isec.rh.container.actualfmt,
|
|
1270
|
+
)
|
|
1271
|
+
logger.info(
|
|
1272
|
+
"%s copied as %s. For initialisation purposes only.",
|
|
1273
|
+
isec.rh.container.localpath(),
|
|
1274
|
+
i_init,
|
|
1275
|
+
)
|
|
907
1276
|
|
|
908
1277
|
# Create all links well in advance
|
|
909
|
-
for i, (iinputs, iguesses) in enumerate(
|
|
910
|
-
|
|
1278
|
+
for i, (iinputs, iguesses) in enumerate(
|
|
1279
|
+
zip(self.inputs.tododata, self.inputs.guessdata)
|
|
1280
|
+
):
|
|
911
1281
|
for iprefix, isec in iinputs.items():
|
|
912
|
-
self._link_input(
|
|
1282
|
+
self._link_input(
|
|
1283
|
+
iprefix, isec.rh, i, inputs_mapping, outputs_mapping
|
|
1284
|
+
)
|
|
913
1285
|
for iguess in iguesses:
|
|
914
1286
|
self._move_output_guess(iguess, i)
|
|
915
1287
|
|
|
@@ -921,7 +1293,11 @@ class FullPosServer(IFSParallel):
|
|
|
921
1293
|
super().execute(rh, opts)
|
|
922
1294
|
|
|
923
1295
|
# Map all outputs to destination (using io_poll)
|
|
924
|
-
self.io_poll_args = tuple(
|
|
925
|
-
|
|
1296
|
+
self.io_poll_args = tuple(
|
|
1297
|
+
[
|
|
1298
|
+
self._MODELSIDE_OUTPUTPREFIX,
|
|
1299
|
+
self._MODELSIDE_OUTPUTPREFIX_GRIB,
|
|
1300
|
+
]
|
|
1301
|
+
)
|
|
926
1302
|
self._init_poll_and_move(outputs_mapping)
|
|
927
1303
|
self._poll_and_move(outputs_mapping)
|