vortex-nwp 2.0.0b1__py3-none-any.whl → 2.0.0b2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- vortex/__init__.py +59 -45
- vortex/algo/__init__.py +3 -2
- vortex/algo/components.py +940 -614
- vortex/algo/mpitools.py +802 -497
- vortex/algo/serversynctools.py +34 -33
- vortex/config.py +19 -22
- vortex/data/__init__.py +9 -3
- vortex/data/abstractstores.py +593 -655
- vortex/data/containers.py +217 -162
- vortex/data/contents.py +65 -39
- vortex/data/executables.py +93 -102
- vortex/data/flow.py +40 -34
- vortex/data/geometries.py +228 -132
- vortex/data/handlers.py +428 -225
- vortex/data/outflow.py +15 -15
- vortex/data/providers.py +185 -163
- vortex/data/resources.py +48 -42
- vortex/data/stores.py +544 -413
- vortex/gloves.py +114 -87
- vortex/layout/__init__.py +1 -8
- vortex/layout/contexts.py +150 -84
- vortex/layout/dataflow.py +353 -202
- vortex/layout/monitor.py +264 -128
- vortex/nwp/__init__.py +5 -2
- vortex/nwp/algo/__init__.py +14 -5
- vortex/nwp/algo/assim.py +205 -151
- vortex/nwp/algo/clim.py +683 -517
- vortex/nwp/algo/coupling.py +447 -225
- vortex/nwp/algo/eda.py +437 -229
- vortex/nwp/algo/eps.py +403 -231
- vortex/nwp/algo/forecasts.py +420 -271
- vortex/nwp/algo/fpserver.py +683 -307
- vortex/nwp/algo/ifsnaming.py +205 -145
- vortex/nwp/algo/ifsroot.py +210 -122
- vortex/nwp/algo/monitoring.py +132 -76
- vortex/nwp/algo/mpitools.py +321 -191
- vortex/nwp/algo/odbtools.py +617 -353
- vortex/nwp/algo/oopsroot.py +449 -273
- vortex/nwp/algo/oopstests.py +90 -56
- vortex/nwp/algo/request.py +287 -206
- vortex/nwp/algo/stdpost.py +878 -522
- vortex/nwp/data/__init__.py +22 -4
- vortex/nwp/data/assim.py +125 -137
- vortex/nwp/data/boundaries.py +121 -68
- vortex/nwp/data/climfiles.py +193 -211
- vortex/nwp/data/configfiles.py +73 -69
- vortex/nwp/data/consts.py +426 -401
- vortex/nwp/data/ctpini.py +59 -43
- vortex/nwp/data/diagnostics.py +94 -66
- vortex/nwp/data/eda.py +50 -51
- vortex/nwp/data/eps.py +195 -146
- vortex/nwp/data/executables.py +440 -434
- vortex/nwp/data/fields.py +63 -48
- vortex/nwp/data/gridfiles.py +183 -111
- vortex/nwp/data/logs.py +250 -217
- vortex/nwp/data/modelstates.py +180 -151
- vortex/nwp/data/monitoring.py +72 -99
- vortex/nwp/data/namelists.py +254 -202
- vortex/nwp/data/obs.py +400 -308
- vortex/nwp/data/oopsexec.py +22 -20
- vortex/nwp/data/providers.py +90 -65
- vortex/nwp/data/query.py +71 -82
- vortex/nwp/data/stores.py +49 -36
- vortex/nwp/data/surfex.py +136 -137
- vortex/nwp/syntax/__init__.py +1 -1
- vortex/nwp/syntax/stdattrs.py +173 -111
- vortex/nwp/tools/__init__.py +2 -2
- vortex/nwp/tools/addons.py +22 -17
- vortex/nwp/tools/agt.py +24 -12
- vortex/nwp/tools/bdap.py +16 -5
- vortex/nwp/tools/bdcp.py +4 -1
- vortex/nwp/tools/bdm.py +3 -0
- vortex/nwp/tools/bdmp.py +14 -9
- vortex/nwp/tools/conftools.py +728 -378
- vortex/nwp/tools/drhook.py +12 -8
- vortex/nwp/tools/grib.py +65 -39
- vortex/nwp/tools/gribdiff.py +22 -17
- vortex/nwp/tools/ifstools.py +82 -42
- vortex/nwp/tools/igastuff.py +167 -143
- vortex/nwp/tools/mars.py +14 -2
- vortex/nwp/tools/odb.py +234 -125
- vortex/nwp/tools/partitioning.py +61 -37
- vortex/nwp/tools/satrad.py +27 -12
- vortex/nwp/util/async.py +83 -55
- vortex/nwp/util/beacon.py +10 -10
- vortex/nwp/util/diffpygram.py +174 -86
- vortex/nwp/util/ens.py +144 -63
- vortex/nwp/util/hooks.py +30 -19
- vortex/nwp/util/taskdeco.py +28 -24
- vortex/nwp/util/usepygram.py +278 -172
- vortex/nwp/util/usetnt.py +31 -17
- vortex/sessions.py +72 -39
- vortex/syntax/__init__.py +1 -1
- vortex/syntax/stdattrs.py +410 -171
- vortex/syntax/stddeco.py +31 -22
- vortex/toolbox.py +327 -192
- vortex/tools/__init__.py +11 -2
- vortex/tools/actions.py +125 -59
- vortex/tools/addons.py +111 -92
- vortex/tools/arm.py +42 -22
- vortex/tools/compression.py +72 -69
- vortex/tools/date.py +11 -4
- vortex/tools/delayedactions.py +242 -132
- vortex/tools/env.py +75 -47
- vortex/tools/folder.py +342 -171
- vortex/tools/grib.py +311 -149
- vortex/tools/lfi.py +423 -216
- vortex/tools/listings.py +109 -40
- vortex/tools/names.py +218 -156
- vortex/tools/net.py +632 -298
- vortex/tools/parallelism.py +93 -61
- vortex/tools/prestaging.py +55 -31
- vortex/tools/schedulers.py +172 -105
- vortex/tools/services.py +402 -333
- vortex/tools/storage.py +293 -358
- vortex/tools/surfex.py +24 -24
- vortex/tools/systems.py +1211 -631
- vortex/tools/targets.py +156 -100
- vortex/util/__init__.py +1 -1
- vortex/util/config.py +377 -327
- vortex/util/empty.py +2 -2
- vortex/util/helpers.py +56 -24
- vortex/util/introspection.py +18 -12
- vortex/util/iosponge.py +8 -4
- vortex/util/roles.py +4 -6
- vortex/util/storefunctions.py +39 -13
- vortex/util/structs.py +3 -3
- vortex/util/worker.py +29 -17
- vortex_nwp-2.0.0b2.dist-info/METADATA +66 -0
- vortex_nwp-2.0.0b2.dist-info/RECORD +142 -0
- {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.0.0b2.dist-info}/WHEEL +1 -1
- vortex/layout/appconf.py +0 -109
- vortex/layout/jobs.py +0 -1276
- vortex/layout/nodes.py +0 -1424
- vortex/layout/subjobs.py +0 -464
- vortex_nwp-2.0.0b1.dist-info/METADATA +0 -50
- vortex_nwp-2.0.0b1.dist-info/RECORD +0 -146
- {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.0.0b2.dist-info}/LICENSE +0 -0
- {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.0.0b2.dist-info}/top_level.txt +0 -0
vortex/tools/grib.py
CHANGED
|
@@ -19,7 +19,10 @@ import footprints
|
|
|
19
19
|
|
|
20
20
|
from . import addons
|
|
21
21
|
from vortex.config import get_from_config_w_default
|
|
22
|
-
from vortex.algo.components import
|
|
22
|
+
from vortex.algo.components import (
|
|
23
|
+
AlgoComponentDecoMixin,
|
|
24
|
+
algo_component_deco_mixin_autodoc,
|
|
25
|
+
)
|
|
23
26
|
from vortex.tools.net import DEFAULT_FTP_PORT
|
|
24
27
|
|
|
25
28
|
#: No automatic export
|
|
@@ -30,7 +33,7 @@ logger = loggers.getLogger(__name__)
|
|
|
30
33
|
|
|
31
34
|
def use_in_shell(sh, **kw):
|
|
32
35
|
"""Extend current shell with the LFI interface defined by optional arguments."""
|
|
33
|
-
kw[
|
|
36
|
+
kw["shell"] = sh
|
|
34
37
|
return footprints.proxy.addon(**kw)
|
|
35
38
|
|
|
36
39
|
|
|
@@ -38,13 +41,14 @@ class GRIB_Tool(addons.FtrawEnableAddon):
|
|
|
38
41
|
"""
|
|
39
42
|
Handle multipart-GRIB files properly.
|
|
40
43
|
"""
|
|
44
|
+
|
|
41
45
|
_footprint = dict(
|
|
42
|
-
info
|
|
43
|
-
attr
|
|
44
|
-
kind
|
|
45
|
-
values
|
|
46
|
+
info="Default GRIB system interface",
|
|
47
|
+
attr=dict(
|
|
48
|
+
kind=dict(
|
|
49
|
+
values=["grib"],
|
|
46
50
|
),
|
|
47
|
-
)
|
|
51
|
+
),
|
|
48
52
|
)
|
|
49
53
|
|
|
50
54
|
def _std_grib_index_get(self, source):
|
|
@@ -55,24 +59,33 @@ class GRIB_Tool(addons.FtrawEnableAddon):
|
|
|
55
59
|
xgrib_index_get = _std_grib_index_get
|
|
56
60
|
|
|
57
61
|
def _std_grib_index_write(self, destination, gribpaths):
|
|
58
|
-
gribparts = [
|
|
59
|
-
|
|
62
|
+
gribparts = [
|
|
63
|
+
str(urlparse.urlunparse(("file", "", path, "", "", "")))
|
|
64
|
+
for path in gribpaths
|
|
65
|
+
]
|
|
60
66
|
tmpfile = self.sh.safe_fileaddsuffix(destination)
|
|
61
|
-
with open(tmpfile,
|
|
62
|
-
fd.write(
|
|
67
|
+
with open(tmpfile, "w") as fd:
|
|
68
|
+
fd.write("\n".join(gribparts))
|
|
63
69
|
return self.sh.move(tmpfile, destination)
|
|
64
70
|
|
|
65
71
|
def is_xgrib(self, source):
|
|
66
72
|
"""Check if the given ``source`` is a multipart-GRIB file."""
|
|
67
73
|
rc = False
|
|
68
74
|
if source and isinstance(source, str) and self.sh.path.exists(source):
|
|
69
|
-
with open(source,
|
|
70
|
-
rc = fd.read(7) == b
|
|
75
|
+
with open(source, "rb") as fd:
|
|
76
|
+
rc = fd.read(7) == b"file://"
|
|
71
77
|
return rc
|
|
72
78
|
|
|
73
|
-
def _backend_cp(
|
|
74
|
-
|
|
75
|
-
|
|
79
|
+
def _backend_cp(
|
|
80
|
+
self, source, destination, smartcp_threshold=0, intent="in"
|
|
81
|
+
):
|
|
82
|
+
return self.sh.cp(
|
|
83
|
+
source,
|
|
84
|
+
destination,
|
|
85
|
+
smartcp_threshold=smartcp_threshold,
|
|
86
|
+
intent=intent,
|
|
87
|
+
smartcp=True,
|
|
88
|
+
)
|
|
76
89
|
|
|
77
90
|
def _backend_rm(self, *args):
|
|
78
91
|
return self.sh.rm(*args)
|
|
@@ -103,8 +116,15 @@ class GRIB_Tool(addons.FtrawEnableAddon):
|
|
|
103
116
|
|
|
104
117
|
grib_rm = grib_remove = _std_remove
|
|
105
118
|
|
|
106
|
-
def _std_copy(
|
|
107
|
-
|
|
119
|
+
def _std_copy(
|
|
120
|
+
self,
|
|
121
|
+
source,
|
|
122
|
+
destination,
|
|
123
|
+
smartcp_threshold=0,
|
|
124
|
+
intent="in",
|
|
125
|
+
pack=False,
|
|
126
|
+
silent=False,
|
|
127
|
+
):
|
|
108
128
|
"""Extended copy for (possibly) multi GRIB file."""
|
|
109
129
|
# Might be multipart
|
|
110
130
|
if self.is_xgrib(source):
|
|
@@ -112,22 +132,38 @@ class GRIB_Tool(addons.FtrawEnableAddon):
|
|
|
112
132
|
if isinstance(destination, str) and not pack:
|
|
113
133
|
with self.sh.mute_stderr():
|
|
114
134
|
idx = self._std_grib_index_get(source)
|
|
115
|
-
destdir = self.sh.path.abspath(
|
|
135
|
+
destdir = self.sh.path.abspath(
|
|
136
|
+
self.sh.path.expanduser(destination) + ".d"
|
|
137
|
+
)
|
|
116
138
|
rc = rc and self.sh.mkdir(destdir)
|
|
117
139
|
target_idx = list()
|
|
118
|
-
for
|
|
119
|
-
target_idx.append(
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
140
|
+
for i, a_mpart in enumerate(idx):
|
|
141
|
+
target_idx.append(
|
|
142
|
+
self.sh.path.join(
|
|
143
|
+
destdir, "GRIB_mpart{:06d}".format(i)
|
|
144
|
+
)
|
|
145
|
+
)
|
|
146
|
+
rc = rc and self._backend_cp(
|
|
147
|
+
a_mpart,
|
|
148
|
+
target_idx[-1],
|
|
149
|
+
smartcp_threshold=smartcp_threshold,
|
|
150
|
+
intent=intent,
|
|
151
|
+
)
|
|
152
|
+
rc = rc and self._std_grib_index_write(
|
|
153
|
+
destination, target_idx
|
|
154
|
+
)
|
|
155
|
+
if intent == "in":
|
|
124
156
|
self.sh.chmod(destination, 0o444)
|
|
125
157
|
else:
|
|
126
158
|
rc = rc and self.xgrib_pack(source, destination)
|
|
127
159
|
else:
|
|
128
160
|
# Usual file or file descriptor
|
|
129
|
-
rc = self._backend_cp(
|
|
130
|
-
|
|
161
|
+
rc = self._backend_cp(
|
|
162
|
+
source,
|
|
163
|
+
destination,
|
|
164
|
+
smartcp_threshold=smartcp_threshold,
|
|
165
|
+
intent=intent,
|
|
166
|
+
)
|
|
131
167
|
return rc
|
|
132
168
|
|
|
133
169
|
grib_cp = grib_copy = _std_copy
|
|
@@ -136,7 +172,7 @@ class GRIB_Tool(addons.FtrawEnableAddon):
|
|
|
136
172
|
"""Extended mv for (possibly) multi GRIB file."""
|
|
137
173
|
# Might be multipart
|
|
138
174
|
if self.is_xgrib(source):
|
|
139
|
-
intent =
|
|
175
|
+
intent = "inout" if self.sh.access(source, self.sh.W_OK) else "in"
|
|
140
176
|
rc = self._std_copy(source, destination, intent=intent)
|
|
141
177
|
rc = rc and self._std_remove(source)
|
|
142
178
|
else:
|
|
@@ -146,7 +182,9 @@ class GRIB_Tool(addons.FtrawEnableAddon):
|
|
|
146
182
|
grib_mv = grib_move = _std_move
|
|
147
183
|
|
|
148
184
|
def _pack_stream(self, source, stdout=True):
|
|
149
|
-
cmd = [
|
|
185
|
+
cmd = [
|
|
186
|
+
"cat",
|
|
187
|
+
]
|
|
150
188
|
cmd.extend(self._std_grib_index_get(source))
|
|
151
189
|
return self.sh.popen(cmd, stdout=stdout, bufsize=8192)
|
|
152
190
|
|
|
@@ -159,14 +197,14 @@ class GRIB_Tool(addons.FtrawEnableAddon):
|
|
|
159
197
|
total += size
|
|
160
198
|
return total
|
|
161
199
|
|
|
162
|
-
def xgrib_pack(self, source, destination, intent=
|
|
200
|
+
def xgrib_pack(self, source, destination, intent="in"):
|
|
163
201
|
"""Manually pack a multi GRIB."""
|
|
164
202
|
if isinstance(destination, str):
|
|
165
203
|
tmpfile = self.sh.safe_fileaddsuffix(destination)
|
|
166
|
-
with open(tmpfile,
|
|
204
|
+
with open(tmpfile, "wb") as fd:
|
|
167
205
|
p = self._pack_stream(source, stdout=fd)
|
|
168
206
|
self.sh.pclose(p)
|
|
169
|
-
if intent ==
|
|
207
|
+
if intent == "in":
|
|
170
208
|
self.sh.chmod(tmpfile, 0o444)
|
|
171
209
|
return self.sh.move(tmpfile, destination)
|
|
172
210
|
else:
|
|
@@ -177,13 +215,16 @@ class GRIB_Tool(addons.FtrawEnableAddon):
|
|
|
177
215
|
def _std_forcepack(self, source, destination=None):
|
|
178
216
|
"""Returned a path to a packed data."""
|
|
179
217
|
if self.is_xgrib(source):
|
|
180
|
-
destination = (
|
|
181
|
-
|
|
218
|
+
destination = (
|
|
219
|
+
destination
|
|
220
|
+
if destination
|
|
221
|
+
else self.sh.safe_fileaddsuffix(source)
|
|
222
|
+
)
|
|
182
223
|
if not self.sh.path.exists(destination):
|
|
183
224
|
if self.xgrib_pack(source, destination):
|
|
184
225
|
return destination
|
|
185
226
|
else:
|
|
186
|
-
raise OSError(
|
|
227
|
+
raise OSError("XGrib packing failed")
|
|
187
228
|
else:
|
|
188
229
|
return destination
|
|
189
230
|
else:
|
|
@@ -191,8 +232,16 @@ class GRIB_Tool(addons.FtrawEnableAddon):
|
|
|
191
232
|
|
|
192
233
|
grib_forcepack = _std_forcepack
|
|
193
234
|
|
|
194
|
-
def _std_ftput(
|
|
195
|
-
|
|
235
|
+
def _std_ftput(
|
|
236
|
+
self,
|
|
237
|
+
source,
|
|
238
|
+
destination,
|
|
239
|
+
hostname=None,
|
|
240
|
+
logname=None,
|
|
241
|
+
port=DEFAULT_FTP_PORT,
|
|
242
|
+
cpipeline=None,
|
|
243
|
+
sync=False,
|
|
244
|
+
):
|
|
196
245
|
"""On the fly packing and ftp."""
|
|
197
246
|
if self.is_xgrib(source):
|
|
198
247
|
if cpipeline is not None:
|
|
@@ -202,19 +251,35 @@ class GRIB_Tool(addons.FtrawEnableAddon):
|
|
|
202
251
|
if ftp:
|
|
203
252
|
packed_size = self._packed_size(source)
|
|
204
253
|
p = self._pack_stream(source)
|
|
205
|
-
rc = ftp.put(
|
|
254
|
+
rc = ftp.put(
|
|
255
|
+
p.stdout, destination, size=packed_size, exact=True
|
|
256
|
+
)
|
|
206
257
|
self.sh.pclose(p)
|
|
207
258
|
ftp.close()
|
|
208
259
|
else:
|
|
209
260
|
rc = False
|
|
210
261
|
return rc
|
|
211
262
|
else:
|
|
212
|
-
return self.sh.ftput(
|
|
213
|
-
|
|
214
|
-
|
|
263
|
+
return self.sh.ftput(
|
|
264
|
+
source,
|
|
265
|
+
destination,
|
|
266
|
+
hostname=hostname,
|
|
267
|
+
logname=logname,
|
|
268
|
+
port=port,
|
|
269
|
+
cpipeline=cpipeline,
|
|
270
|
+
sync=sync,
|
|
271
|
+
)
|
|
215
272
|
|
|
216
|
-
def _std_rawftput(
|
|
217
|
-
|
|
273
|
+
def _std_rawftput(
|
|
274
|
+
self,
|
|
275
|
+
source,
|
|
276
|
+
destination,
|
|
277
|
+
hostname=None,
|
|
278
|
+
logname=None,
|
|
279
|
+
port=None,
|
|
280
|
+
cpipeline=None,
|
|
281
|
+
sync=False,
|
|
282
|
+
):
|
|
218
283
|
"""Use ftserv as much as possible."""
|
|
219
284
|
if self.is_xgrib(source):
|
|
220
285
|
if cpipeline is not None:
|
|
@@ -222,52 +287,82 @@ class GRIB_Tool(addons.FtrawEnableAddon):
|
|
|
222
287
|
if self.sh.ftraw and self.rawftshell is not None:
|
|
223
288
|
# Copy the GRIB pieces individually
|
|
224
289
|
pieces = self.xgrib_index_get(source)
|
|
225
|
-
newsources = [
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
290
|
+
newsources = [
|
|
291
|
+
str(self.sh.copy2ftspool(piece)) for piece in pieces
|
|
292
|
+
]
|
|
293
|
+
request = newsources[0] + ".request"
|
|
294
|
+
with open(request, "w") as request_fh:
|
|
295
|
+
request_fh.writelines("\n".join(newsources))
|
|
229
296
|
self.sh.readonly(request)
|
|
230
|
-
rc = self.sh.ftserv_put(
|
|
231
|
-
|
|
232
|
-
|
|
297
|
+
rc = self.sh.ftserv_put(
|
|
298
|
+
request,
|
|
299
|
+
destination,
|
|
300
|
+
hostname=hostname,
|
|
301
|
+
logname=logname,
|
|
302
|
+
port=port,
|
|
303
|
+
specialshell=self.rawftshell,
|
|
304
|
+
sync=sync,
|
|
305
|
+
)
|
|
233
306
|
self.sh.rm(request)
|
|
234
307
|
return rc
|
|
235
308
|
else:
|
|
236
309
|
if port is None:
|
|
237
310
|
port = DEFAULT_FTP_PORT
|
|
238
|
-
return self._std_ftput(
|
|
239
|
-
|
|
240
|
-
|
|
311
|
+
return self._std_ftput(
|
|
312
|
+
source,
|
|
313
|
+
destination,
|
|
314
|
+
hostname=hostname,
|
|
315
|
+
logname=logname,
|
|
316
|
+
port=port,
|
|
317
|
+
sync=sync,
|
|
318
|
+
)
|
|
241
319
|
else:
|
|
242
|
-
return self.sh.rawftput(
|
|
243
|
-
|
|
244
|
-
|
|
320
|
+
return self.sh.rawftput(
|
|
321
|
+
source,
|
|
322
|
+
destination,
|
|
323
|
+
hostname=hostname,
|
|
324
|
+
logname=logname,
|
|
325
|
+
port=port,
|
|
326
|
+
cpipeline=cpipeline,
|
|
327
|
+
sync=sync,
|
|
328
|
+
)
|
|
245
329
|
|
|
246
330
|
grib_ftput = _std_ftput
|
|
247
331
|
grib_rawftput = _std_rawftput
|
|
248
332
|
|
|
249
|
-
def _std_scpput(
|
|
333
|
+
def _std_scpput(
|
|
334
|
+
self, source, destination, hostname, logname=None, cpipeline=None
|
|
335
|
+
):
|
|
250
336
|
"""On the fly packing and scp."""
|
|
251
337
|
if self.is_xgrib(source):
|
|
252
338
|
if cpipeline is not None:
|
|
253
339
|
raise OSError("It's not allowed to compress xgrib files.")
|
|
254
|
-
logname = self.sh.fix_ftuser(
|
|
340
|
+
logname = self.sh.fix_ftuser(
|
|
341
|
+
hostname, logname, fatal=False, defaults_to_user=False
|
|
342
|
+
)
|
|
255
343
|
ssh = self.sh.ssh(hostname, logname)
|
|
256
344
|
permissions = ssh.get_permissions(source)
|
|
257
345
|
# remove the .d companion directory (scp_stream removes the destination)
|
|
258
346
|
# go on on failure : the .d lingers on, but the grib will be self-contained
|
|
259
|
-
ssh.remove(destination +
|
|
347
|
+
ssh.remove(destination + ".d")
|
|
260
348
|
p = self._pack_stream(source)
|
|
261
|
-
rc = ssh.scpput_stream(
|
|
349
|
+
rc = ssh.scpput_stream(
|
|
350
|
+
p.stdout, destination, permissions=permissions
|
|
351
|
+
)
|
|
262
352
|
self.sh.pclose(p)
|
|
263
353
|
return rc
|
|
264
354
|
else:
|
|
265
|
-
return self.sh.scpput(
|
|
266
|
-
|
|
355
|
+
return self.sh.scpput(
|
|
356
|
+
source,
|
|
357
|
+
destination,
|
|
358
|
+
hostname,
|
|
359
|
+
logname=logname,
|
|
360
|
+
cpipeline=cpipeline,
|
|
361
|
+
)
|
|
267
362
|
|
|
268
363
|
grib_scpput = _std_scpput
|
|
269
364
|
|
|
270
|
-
@addons.require_external_addon(
|
|
365
|
+
@addons.require_external_addon("ecfs")
|
|
271
366
|
def grib_ecfsput(self, source, target, cpipeline=None, options=None):
|
|
272
367
|
"""Put a grib resource using ECfs.
|
|
273
368
|
|
|
@@ -282,25 +377,33 @@ class GRIB_Tool(addons.FtrawEnableAddon):
|
|
|
282
377
|
raise OSError("It's not allowed to compress xgrib files.")
|
|
283
378
|
psource = self.sh.safe_fileaddsuffix(source)
|
|
284
379
|
try:
|
|
285
|
-
rc = self.xgrib_pack(source=source,
|
|
286
|
-
destination=psource)
|
|
380
|
+
rc = self.xgrib_pack(source=source, destination=psource)
|
|
287
381
|
dict_args = dict()
|
|
288
382
|
if rc:
|
|
289
|
-
rc, dict_args = self.sh.ecfsput(
|
|
290
|
-
|
|
291
|
-
|
|
383
|
+
rc, dict_args = self.sh.ecfsput(
|
|
384
|
+
source=psource, target=target, options=options
|
|
385
|
+
)
|
|
292
386
|
finally:
|
|
293
387
|
self.sh.rm(psource)
|
|
294
388
|
return rc, dict_args
|
|
295
389
|
else:
|
|
296
|
-
return self.sh.ecfsput(
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
390
|
+
return self.sh.ecfsput(
|
|
391
|
+
source=source,
|
|
392
|
+
target=target,
|
|
393
|
+
options=options,
|
|
394
|
+
cpipeline=cpipeline,
|
|
395
|
+
)
|
|
396
|
+
|
|
397
|
+
@addons.require_external_addon("ectrans")
|
|
398
|
+
def grib_ectransput(
|
|
399
|
+
self,
|
|
400
|
+
source,
|
|
401
|
+
target,
|
|
402
|
+
gateway=None,
|
|
403
|
+
remote=None,
|
|
404
|
+
cpipeline=None,
|
|
405
|
+
sync=False,
|
|
406
|
+
):
|
|
304
407
|
"""Put a grib resource using ECtrans.
|
|
305
408
|
|
|
306
409
|
:param source: source file
|
|
@@ -316,25 +419,28 @@ class GRIB_Tool(addons.FtrawEnableAddon):
|
|
|
316
419
|
raise OSError("It's not allowed to compress xgrib files.")
|
|
317
420
|
psource = self.sh.safe_fileaddsuffix(source)
|
|
318
421
|
try:
|
|
319
|
-
rc = self.xgrib_pack(source=source,
|
|
320
|
-
destination=psource)
|
|
422
|
+
rc = self.xgrib_pack(source=source, destination=psource)
|
|
321
423
|
dict_args = dict()
|
|
322
424
|
if rc:
|
|
323
|
-
rc, dict_args = self.sh.raw_ectransput(
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
425
|
+
rc, dict_args = self.sh.raw_ectransput(
|
|
426
|
+
source=psource,
|
|
427
|
+
target=target,
|
|
428
|
+
gateway=gateway,
|
|
429
|
+
remote=remote,
|
|
430
|
+
sync=sync,
|
|
431
|
+
)
|
|
328
432
|
finally:
|
|
329
433
|
self.sh.rm(psource)
|
|
330
434
|
return rc, dict_args
|
|
331
435
|
else:
|
|
332
|
-
return self.sh.ectransput(
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
436
|
+
return self.sh.ectransput(
|
|
437
|
+
source=source,
|
|
438
|
+
target=target,
|
|
439
|
+
gateway=gateway,
|
|
440
|
+
remote=remote,
|
|
441
|
+
cpipeline=cpipeline,
|
|
442
|
+
sync=sync,
|
|
443
|
+
)
|
|
338
444
|
|
|
339
445
|
|
|
340
446
|
@algo_component_deco_mixin_autodoc
|
|
@@ -356,23 +462,37 @@ class EcGribDecoMixin(AlgoComponentDecoMixin):
|
|
|
356
462
|
gribapi_lib = None
|
|
357
463
|
if rh is not None:
|
|
358
464
|
if not isinstance(rh, (list, tuple)):
|
|
359
|
-
rh = [
|
|
465
|
+
rh = [
|
|
466
|
+
rh,
|
|
467
|
+
]
|
|
360
468
|
for a_rh in rh:
|
|
361
469
|
libs = self.system.ldd(a_rh.container.localpath())
|
|
362
470
|
a_eccodes_lib = None
|
|
363
471
|
a_gribapi_lib = None
|
|
364
472
|
for lib, path in libs.items():
|
|
365
|
-
if re.match(
|
|
473
|
+
if re.match(
|
|
474
|
+
r"^libeccodes(?:-[.0-9]+)?\.so(?:\.[.0-9]+)?$", lib
|
|
475
|
+
):
|
|
366
476
|
a_eccodes_lib = path
|
|
367
|
-
if re.match(
|
|
477
|
+
if re.match(
|
|
478
|
+
r"^libgrib_api(?:-[.0-9]+)?\.so(?:\.[.0-9]+)?$", lib
|
|
479
|
+
):
|
|
368
480
|
a_gribapi_lib = path
|
|
369
481
|
if a_eccodes_lib:
|
|
370
|
-
self.algoassert(
|
|
371
|
-
|
|
482
|
+
self.algoassert(
|
|
483
|
+
eccodes_lib is None or (eccodes_lib == a_eccodes_lib),
|
|
484
|
+
"ecCodes library inconsistency (rh: {!s})".format(
|
|
485
|
+
a_rh
|
|
486
|
+
),
|
|
487
|
+
)
|
|
372
488
|
eccodes_lib = a_eccodes_lib
|
|
373
489
|
if a_gribapi_lib:
|
|
374
|
-
self.algoassert(
|
|
375
|
-
|
|
490
|
+
self.algoassert(
|
|
491
|
+
gribapi_lib is None or (gribapi_lib == a_gribapi_lib),
|
|
492
|
+
"grib_api library inconsistency (rh: {!s})".format(
|
|
493
|
+
a_rh
|
|
494
|
+
),
|
|
495
|
+
)
|
|
376
496
|
gribapi_lib = a_gribapi_lib
|
|
377
497
|
return eccodes_lib, gribapi_lib
|
|
378
498
|
|
|
@@ -380,71 +500,99 @@ class EcGribDecoMixin(AlgoComponentDecoMixin):
|
|
|
380
500
|
"""Add axtra definitions/samples to the library path."""
|
|
381
501
|
for gdef in self.context.sequence.effective_inputs(role=a_role):
|
|
382
502
|
local_path = gdef.rh.container.localpath()
|
|
383
|
-
new_path = (
|
|
384
|
-
|
|
503
|
+
new_path = (
|
|
504
|
+
local_path
|
|
505
|
+
if self.system.path.isdir(local_path)
|
|
506
|
+
else self.system.path.dirname(local_path)
|
|
507
|
+
)
|
|
385
508
|
# NB: Grib-API doesn't understand relative paths...
|
|
386
509
|
new_path = self.system.path.abspath(new_path)
|
|
387
510
|
self.env.setgenericpath(a_var, new_path, pos=0)
|
|
388
511
|
|
|
389
512
|
def _gribapi_envsetup(self, gribapi_lib):
|
|
390
513
|
"""Setup environment variables for grib_api."""
|
|
391
|
-
defvar =
|
|
392
|
-
samplevar =
|
|
514
|
+
defvar = "GRIB_DEFINITION_PATH"
|
|
515
|
+
samplevar = "GRIB_SAMPLES_PATH"
|
|
393
516
|
if gribapi_lib is not None:
|
|
394
517
|
gribapi_root = self.system.path.dirname(gribapi_lib)
|
|
395
518
|
gribapi_root = self.system.path.split(gribapi_root)[0]
|
|
396
|
-
gribapi_share = self.system.path.join(
|
|
519
|
+
gribapi_share = self.system.path.join(
|
|
520
|
+
gribapi_root, "share", "grib_api"
|
|
521
|
+
)
|
|
397
522
|
if defvar not in self.env:
|
|
398
523
|
# This one is for compatibility with old versions of the gribapi !
|
|
399
|
-
self.env.setgenericpath(
|
|
400
|
-
|
|
524
|
+
self.env.setgenericpath(
|
|
525
|
+
defvar,
|
|
526
|
+
self.system.path.join(
|
|
527
|
+
gribapi_root, "share", "definitions"
|
|
528
|
+
),
|
|
529
|
+
)
|
|
401
530
|
# This should be the lastest one:
|
|
402
|
-
self.env.setgenericpath(
|
|
403
|
-
|
|
531
|
+
self.env.setgenericpath(
|
|
532
|
+
defvar, self.system.path.join(gribapi_share, "definitions")
|
|
533
|
+
)
|
|
404
534
|
if samplevar not in self.env:
|
|
405
535
|
# This one is for compatibility with old versions of the gribapi !
|
|
406
|
-
self.env.setgenericpath(
|
|
407
|
-
|
|
536
|
+
self.env.setgenericpath(
|
|
537
|
+
samplevar,
|
|
538
|
+
self.system.path.join(
|
|
539
|
+
gribapi_root, "ifs_samples", "grib1"
|
|
540
|
+
),
|
|
541
|
+
)
|
|
408
542
|
# This should be the lastest one:
|
|
409
|
-
self.env.setgenericpath(
|
|
410
|
-
|
|
543
|
+
self.env.setgenericpath(
|
|
544
|
+
samplevar,
|
|
545
|
+
self.system.path.join(
|
|
546
|
+
gribapi_share, "ifs_samples", "grib1"
|
|
547
|
+
),
|
|
548
|
+
)
|
|
411
549
|
else:
|
|
412
550
|
# Use the default GRIB-API config if the ldd approach fails
|
|
413
|
-
self.export(
|
|
551
|
+
self.export("gribapi")
|
|
414
552
|
return defvar, samplevar
|
|
415
553
|
|
|
416
554
|
def gribapi_setup(self, rh, opts):
|
|
417
555
|
"""Setup the grib_api related stuff."""
|
|
418
556
|
_, gribapi_lib = self._ecgrib_libs_detext(rh)
|
|
419
557
|
defvar, samplevar = self._gribapi_envsetup(gribapi_lib)
|
|
420
|
-
self._ecgrib_additional_config(
|
|
421
|
-
self._ecgrib_additional_config(
|
|
558
|
+
self._ecgrib_additional_config("AdditionalGribAPIDefinitions", defvar)
|
|
559
|
+
self._ecgrib_additional_config("AdditionalGribAPISamples", samplevar)
|
|
422
560
|
# Recap
|
|
423
561
|
for a_var in (defvar, samplevar):
|
|
424
|
-
logger.info(
|
|
562
|
+
logger.info(
|
|
563
|
+
"After gribapi_setup %s = %s", a_var, self.env.getvar(a_var)
|
|
564
|
+
)
|
|
425
565
|
|
|
426
566
|
def _eccodes_envsetup(self, eccodes_lib):
|
|
427
567
|
"""Setup environment variables for ecCodes."""
|
|
428
|
-
dep_warn = (
|
|
429
|
-
|
|
568
|
+
dep_warn = (
|
|
569
|
+
"%s is left unconfigured because the old grib_api's variable is defined."
|
|
570
|
+
+ "Please remove that !"
|
|
571
|
+
)
|
|
430
572
|
eccodes_root = self.system.path.dirname(eccodes_lib)
|
|
431
573
|
eccodes_root = self.system.path.split(eccodes_root)[0]
|
|
432
|
-
eccodes_share = self.system.path.join(eccodes_root,
|
|
433
|
-
defvar =
|
|
574
|
+
eccodes_share = self.system.path.join(eccodes_root, "share", "eccodes")
|
|
575
|
+
defvar = "ECCODES_DEFINITION_PATH"
|
|
434
576
|
if defvar not in self.env:
|
|
435
|
-
if
|
|
577
|
+
if "GRIB_DEFINITION_PATH" in self.env:
|
|
436
578
|
logger.warning(dep_warn, defvar)
|
|
437
|
-
defvar =
|
|
579
|
+
defvar = "GRIB_DEFINITION_PATH"
|
|
438
580
|
else:
|
|
439
|
-
self.env.setgenericpath(
|
|
440
|
-
|
|
581
|
+
self.env.setgenericpath(
|
|
582
|
+
defvar, self.system.path.join(eccodes_share, "definitions")
|
|
583
|
+
)
|
|
584
|
+
samplevar = "ECCODES_SAMPLES_PATH"
|
|
441
585
|
if samplevar not in self.env:
|
|
442
|
-
if
|
|
586
|
+
if "GRIB_SAMPLES_PATH" in self.env:
|
|
443
587
|
logger.warning(dep_warn, samplevar)
|
|
444
|
-
samplevar =
|
|
588
|
+
samplevar = "GRIB_SAMPLES_PATH"
|
|
445
589
|
else:
|
|
446
|
-
self.env.setgenericpath(
|
|
447
|
-
|
|
590
|
+
self.env.setgenericpath(
|
|
591
|
+
samplevar,
|
|
592
|
+
self.system.path.join(
|
|
593
|
+
eccodes_share, "ifs_samples", "grib1"
|
|
594
|
+
),
|
|
595
|
+
)
|
|
448
596
|
return defvar, samplevar
|
|
449
597
|
|
|
450
598
|
def eccodes_setup(self, rh, opts, compat=False, fatal=True):
|
|
@@ -462,26 +610,38 @@ class EcGribDecoMixin(AlgoComponentDecoMixin):
|
|
|
462
610
|
defvar, samplevar = self._gribapi_envsetup(gribapi_lib)
|
|
463
611
|
else:
|
|
464
612
|
if fatal:
|
|
465
|
-
raise RuntimeError(
|
|
613
|
+
raise RuntimeError(
|
|
614
|
+
"No suitable configuration found for ecCodes."
|
|
615
|
+
)
|
|
466
616
|
else:
|
|
467
617
|
logger.error("ecCodes was not found !")
|
|
468
618
|
return
|
|
469
619
|
# Then, inspect the context to look for customised search paths
|
|
470
|
-
self._ecgrib_additional_config(
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
620
|
+
self._ecgrib_additional_config(
|
|
621
|
+
("AdditionalGribAPIDefinitions", "AdditionalEcCodesDefinitions"),
|
|
622
|
+
defvar,
|
|
623
|
+
)
|
|
624
|
+
self._ecgrib_additional_config(
|
|
625
|
+
("AdditionalGribAPISamples", "AdditionalEcCodesSamples"), samplevar
|
|
626
|
+
)
|
|
474
627
|
# Recap
|
|
475
628
|
for a_var in (defvar, samplevar):
|
|
476
|
-
logger.info(
|
|
477
|
-
|
|
629
|
+
logger.info(
|
|
630
|
+
"After eccodes_setup (compat=%s) : %s = %s",
|
|
631
|
+
str(compat),
|
|
632
|
+
a_var,
|
|
633
|
+
self.env.getvar(a_var),
|
|
634
|
+
)
|
|
478
635
|
|
|
479
636
|
def _ecgrib_mixin_setup(self, rh, opts):
|
|
480
|
-
self.eccodes_setup(
|
|
481
|
-
|
|
482
|
-
|
|
637
|
+
self.eccodes_setup(
|
|
638
|
+
rh,
|
|
639
|
+
opts,
|
|
640
|
+
compat=self._ECGRIB_SETUP_COMPAT,
|
|
641
|
+
fatal=self._ECGRIB_SETUP_FATAL,
|
|
642
|
+
)
|
|
483
643
|
|
|
484
|
-
_MIXIN_PREPARE_HOOKS = (_ecgrib_mixin_setup,
|
|
644
|
+
_MIXIN_PREPARE_HOOKS = (_ecgrib_mixin_setup,)
|
|
485
645
|
|
|
486
646
|
|
|
487
647
|
class GRIBAPI_Tool(addons.Addon):
|
|
@@ -490,12 +650,12 @@ class GRIBAPI_Tool(addons.Addon):
|
|
|
490
650
|
"""
|
|
491
651
|
|
|
492
652
|
_footprint = dict(
|
|
493
|
-
info
|
|
494
|
-
attr
|
|
495
|
-
kind
|
|
496
|
-
values
|
|
653
|
+
info="Default GRIBAPI system interface",
|
|
654
|
+
attr=dict(
|
|
655
|
+
kind=dict(
|
|
656
|
+
values=["gribapi"],
|
|
497
657
|
),
|
|
498
|
-
)
|
|
658
|
+
),
|
|
499
659
|
)
|
|
500
660
|
|
|
501
661
|
def __init__(self, *args, **kw):
|
|
@@ -513,17 +673,19 @@ class GRIBAPI_Tool(addons.Addon):
|
|
|
513
673
|
|
|
514
674
|
def _spawn_wrap(self, cmd, **kw):
|
|
515
675
|
"""Internal method calling standard shell spawn."""
|
|
516
|
-
cmd[0] =
|
|
676
|
+
cmd[0] = "bin" + self.sh.path.sep + cmd[0]
|
|
517
677
|
return super()._spawn_wrap(cmd, **kw)
|
|
518
678
|
|
|
519
679
|
def _actual_diff(self, grib1, grib2, skipkeys, **kw):
|
|
520
680
|
"""Run the actual GRIBAPI command."""
|
|
521
|
-
cmd = [
|
|
522
|
-
kw[
|
|
523
|
-
kw[
|
|
681
|
+
cmd = ["grib_compare", "-r", "-b", ",".join(skipkeys), grib1, grib2]
|
|
682
|
+
kw["fatal"] = False
|
|
683
|
+
kw["output"] = False
|
|
524
684
|
return self._spawn_wrap(cmd, **kw)
|
|
525
685
|
|
|
526
|
-
def grib_diff(
|
|
686
|
+
def grib_diff(
|
|
687
|
+
self, grib1, grib2, skipkeys=("generatingProcessIdentifier",), **kw
|
|
688
|
+
):
|
|
527
689
|
"""
|
|
528
690
|
Difference between two GRIB files (using the GRIB-API)
|
|
529
691
|
|
|
@@ -538,15 +700,15 @@ class GRIBAPI_Tool(addons.Addon):
|
|
|
538
700
|
"""
|
|
539
701
|
|
|
540
702
|
# Are multipart GRIB suported ?
|
|
541
|
-
xgrib_support =
|
|
703
|
+
xgrib_support = "grib" in self.sh.loaded_addons()
|
|
542
704
|
grib1_ori = grib1
|
|
543
705
|
grib2_ori = grib2
|
|
544
706
|
if xgrib_support:
|
|
545
707
|
if self.sh.is_xgrib(grib1):
|
|
546
|
-
grib1 = self.sh.safe_fileaddsuffix(grib1_ori) +
|
|
708
|
+
grib1 = self.sh.safe_fileaddsuffix(grib1_ori) + "_diffcat"
|
|
547
709
|
self.sh.xgrib_pack(grib1_ori, grib1)
|
|
548
710
|
if self.sh.is_xgrib(grib2):
|
|
549
|
-
grib2 = self.sh.safe_fileaddsuffix(grib2_ori) +
|
|
711
|
+
grib2 = self.sh.safe_fileaddsuffix(grib2_ori) + "_diffcat"
|
|
550
712
|
self.sh.xgrib_pack(grib2_ori, grib2)
|
|
551
713
|
|
|
552
714
|
rc = self._actual_diff(grib1, grib2, skipkeys, **kw)
|