vortex-nwp 2.0.0b1__py3-none-any.whl → 2.0.0b2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (139) hide show
  1. vortex/__init__.py +59 -45
  2. vortex/algo/__init__.py +3 -2
  3. vortex/algo/components.py +940 -614
  4. vortex/algo/mpitools.py +802 -497
  5. vortex/algo/serversynctools.py +34 -33
  6. vortex/config.py +19 -22
  7. vortex/data/__init__.py +9 -3
  8. vortex/data/abstractstores.py +593 -655
  9. vortex/data/containers.py +217 -162
  10. vortex/data/contents.py +65 -39
  11. vortex/data/executables.py +93 -102
  12. vortex/data/flow.py +40 -34
  13. vortex/data/geometries.py +228 -132
  14. vortex/data/handlers.py +428 -225
  15. vortex/data/outflow.py +15 -15
  16. vortex/data/providers.py +185 -163
  17. vortex/data/resources.py +48 -42
  18. vortex/data/stores.py +544 -413
  19. vortex/gloves.py +114 -87
  20. vortex/layout/__init__.py +1 -8
  21. vortex/layout/contexts.py +150 -84
  22. vortex/layout/dataflow.py +353 -202
  23. vortex/layout/monitor.py +264 -128
  24. vortex/nwp/__init__.py +5 -2
  25. vortex/nwp/algo/__init__.py +14 -5
  26. vortex/nwp/algo/assim.py +205 -151
  27. vortex/nwp/algo/clim.py +683 -517
  28. vortex/nwp/algo/coupling.py +447 -225
  29. vortex/nwp/algo/eda.py +437 -229
  30. vortex/nwp/algo/eps.py +403 -231
  31. vortex/nwp/algo/forecasts.py +420 -271
  32. vortex/nwp/algo/fpserver.py +683 -307
  33. vortex/nwp/algo/ifsnaming.py +205 -145
  34. vortex/nwp/algo/ifsroot.py +210 -122
  35. vortex/nwp/algo/monitoring.py +132 -76
  36. vortex/nwp/algo/mpitools.py +321 -191
  37. vortex/nwp/algo/odbtools.py +617 -353
  38. vortex/nwp/algo/oopsroot.py +449 -273
  39. vortex/nwp/algo/oopstests.py +90 -56
  40. vortex/nwp/algo/request.py +287 -206
  41. vortex/nwp/algo/stdpost.py +878 -522
  42. vortex/nwp/data/__init__.py +22 -4
  43. vortex/nwp/data/assim.py +125 -137
  44. vortex/nwp/data/boundaries.py +121 -68
  45. vortex/nwp/data/climfiles.py +193 -211
  46. vortex/nwp/data/configfiles.py +73 -69
  47. vortex/nwp/data/consts.py +426 -401
  48. vortex/nwp/data/ctpini.py +59 -43
  49. vortex/nwp/data/diagnostics.py +94 -66
  50. vortex/nwp/data/eda.py +50 -51
  51. vortex/nwp/data/eps.py +195 -146
  52. vortex/nwp/data/executables.py +440 -434
  53. vortex/nwp/data/fields.py +63 -48
  54. vortex/nwp/data/gridfiles.py +183 -111
  55. vortex/nwp/data/logs.py +250 -217
  56. vortex/nwp/data/modelstates.py +180 -151
  57. vortex/nwp/data/monitoring.py +72 -99
  58. vortex/nwp/data/namelists.py +254 -202
  59. vortex/nwp/data/obs.py +400 -308
  60. vortex/nwp/data/oopsexec.py +22 -20
  61. vortex/nwp/data/providers.py +90 -65
  62. vortex/nwp/data/query.py +71 -82
  63. vortex/nwp/data/stores.py +49 -36
  64. vortex/nwp/data/surfex.py +136 -137
  65. vortex/nwp/syntax/__init__.py +1 -1
  66. vortex/nwp/syntax/stdattrs.py +173 -111
  67. vortex/nwp/tools/__init__.py +2 -2
  68. vortex/nwp/tools/addons.py +22 -17
  69. vortex/nwp/tools/agt.py +24 -12
  70. vortex/nwp/tools/bdap.py +16 -5
  71. vortex/nwp/tools/bdcp.py +4 -1
  72. vortex/nwp/tools/bdm.py +3 -0
  73. vortex/nwp/tools/bdmp.py +14 -9
  74. vortex/nwp/tools/conftools.py +728 -378
  75. vortex/nwp/tools/drhook.py +12 -8
  76. vortex/nwp/tools/grib.py +65 -39
  77. vortex/nwp/tools/gribdiff.py +22 -17
  78. vortex/nwp/tools/ifstools.py +82 -42
  79. vortex/nwp/tools/igastuff.py +167 -143
  80. vortex/nwp/tools/mars.py +14 -2
  81. vortex/nwp/tools/odb.py +234 -125
  82. vortex/nwp/tools/partitioning.py +61 -37
  83. vortex/nwp/tools/satrad.py +27 -12
  84. vortex/nwp/util/async.py +83 -55
  85. vortex/nwp/util/beacon.py +10 -10
  86. vortex/nwp/util/diffpygram.py +174 -86
  87. vortex/nwp/util/ens.py +144 -63
  88. vortex/nwp/util/hooks.py +30 -19
  89. vortex/nwp/util/taskdeco.py +28 -24
  90. vortex/nwp/util/usepygram.py +278 -172
  91. vortex/nwp/util/usetnt.py +31 -17
  92. vortex/sessions.py +72 -39
  93. vortex/syntax/__init__.py +1 -1
  94. vortex/syntax/stdattrs.py +410 -171
  95. vortex/syntax/stddeco.py +31 -22
  96. vortex/toolbox.py +327 -192
  97. vortex/tools/__init__.py +11 -2
  98. vortex/tools/actions.py +125 -59
  99. vortex/tools/addons.py +111 -92
  100. vortex/tools/arm.py +42 -22
  101. vortex/tools/compression.py +72 -69
  102. vortex/tools/date.py +11 -4
  103. vortex/tools/delayedactions.py +242 -132
  104. vortex/tools/env.py +75 -47
  105. vortex/tools/folder.py +342 -171
  106. vortex/tools/grib.py +311 -149
  107. vortex/tools/lfi.py +423 -216
  108. vortex/tools/listings.py +109 -40
  109. vortex/tools/names.py +218 -156
  110. vortex/tools/net.py +632 -298
  111. vortex/tools/parallelism.py +93 -61
  112. vortex/tools/prestaging.py +55 -31
  113. vortex/tools/schedulers.py +172 -105
  114. vortex/tools/services.py +402 -333
  115. vortex/tools/storage.py +293 -358
  116. vortex/tools/surfex.py +24 -24
  117. vortex/tools/systems.py +1211 -631
  118. vortex/tools/targets.py +156 -100
  119. vortex/util/__init__.py +1 -1
  120. vortex/util/config.py +377 -327
  121. vortex/util/empty.py +2 -2
  122. vortex/util/helpers.py +56 -24
  123. vortex/util/introspection.py +18 -12
  124. vortex/util/iosponge.py +8 -4
  125. vortex/util/roles.py +4 -6
  126. vortex/util/storefunctions.py +39 -13
  127. vortex/util/structs.py +3 -3
  128. vortex/util/worker.py +29 -17
  129. vortex_nwp-2.0.0b2.dist-info/METADATA +66 -0
  130. vortex_nwp-2.0.0b2.dist-info/RECORD +142 -0
  131. {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.0.0b2.dist-info}/WHEEL +1 -1
  132. vortex/layout/appconf.py +0 -109
  133. vortex/layout/jobs.py +0 -1276
  134. vortex/layout/nodes.py +0 -1424
  135. vortex/layout/subjobs.py +0 -464
  136. vortex_nwp-2.0.0b1.dist-info/METADATA +0 -50
  137. vortex_nwp-2.0.0b1.dist-info/RECORD +0 -146
  138. {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.0.0b2.dist-info}/LICENSE +0 -0
  139. {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.0.0b2.dist-info}/top_level.txt +0 -0
vortex/tools/grib.py CHANGED
@@ -19,7 +19,10 @@ import footprints
19
19
 
20
20
  from . import addons
21
21
  from vortex.config import get_from_config_w_default
22
- from vortex.algo.components import AlgoComponentDecoMixin, algo_component_deco_mixin_autodoc
22
+ from vortex.algo.components import (
23
+ AlgoComponentDecoMixin,
24
+ algo_component_deco_mixin_autodoc,
25
+ )
23
26
  from vortex.tools.net import DEFAULT_FTP_PORT
24
27
 
25
28
  #: No automatic export
@@ -30,7 +33,7 @@ logger = loggers.getLogger(__name__)
30
33
 
31
34
  def use_in_shell(sh, **kw):
32
35
  """Extend current shell with the LFI interface defined by optional arguments."""
33
- kw['shell'] = sh
36
+ kw["shell"] = sh
34
37
  return footprints.proxy.addon(**kw)
35
38
 
36
39
 
@@ -38,13 +41,14 @@ class GRIB_Tool(addons.FtrawEnableAddon):
38
41
  """
39
42
  Handle multipart-GRIB files properly.
40
43
  """
44
+
41
45
  _footprint = dict(
42
- info = 'Default GRIB system interface',
43
- attr = dict(
44
- kind = dict(
45
- values = ['grib'],
46
+ info="Default GRIB system interface",
47
+ attr=dict(
48
+ kind=dict(
49
+ values=["grib"],
46
50
  ),
47
- )
51
+ ),
48
52
  )
49
53
 
50
54
  def _std_grib_index_get(self, source):
@@ -55,24 +59,33 @@ class GRIB_Tool(addons.FtrawEnableAddon):
55
59
  xgrib_index_get = _std_grib_index_get
56
60
 
57
61
  def _std_grib_index_write(self, destination, gribpaths):
58
- gribparts = [str(urlparse.urlunparse(('file', '', path, '', '', '')))
59
- for path in gribpaths]
62
+ gribparts = [
63
+ str(urlparse.urlunparse(("file", "", path, "", "", "")))
64
+ for path in gribpaths
65
+ ]
60
66
  tmpfile = self.sh.safe_fileaddsuffix(destination)
61
- with open(tmpfile, 'w') as fd:
62
- fd.write('\n'.join(gribparts))
67
+ with open(tmpfile, "w") as fd:
68
+ fd.write("\n".join(gribparts))
63
69
  return self.sh.move(tmpfile, destination)
64
70
 
65
71
  def is_xgrib(self, source):
66
72
  """Check if the given ``source`` is a multipart-GRIB file."""
67
73
  rc = False
68
74
  if source and isinstance(source, str) and self.sh.path.exists(source):
69
- with open(source, 'rb') as fd:
70
- rc = fd.read(7) == b'file://'
75
+ with open(source, "rb") as fd:
76
+ rc = fd.read(7) == b"file://"
71
77
  return rc
72
78
 
73
- def _backend_cp(self, source, destination, smartcp_threshold=0, intent='in'):
74
- return self.sh.cp(source, destination,
75
- smartcp_threshold=smartcp_threshold, intent=intent, smartcp=True)
79
+ def _backend_cp(
80
+ self, source, destination, smartcp_threshold=0, intent="in"
81
+ ):
82
+ return self.sh.cp(
83
+ source,
84
+ destination,
85
+ smartcp_threshold=smartcp_threshold,
86
+ intent=intent,
87
+ smartcp=True,
88
+ )
76
89
 
77
90
  def _backend_rm(self, *args):
78
91
  return self.sh.rm(*args)
@@ -103,8 +116,15 @@ class GRIB_Tool(addons.FtrawEnableAddon):
103
116
 
104
117
  grib_rm = grib_remove = _std_remove
105
118
 
106
- def _std_copy(self, source, destination,
107
- smartcp_threshold=0, intent='in', pack=False, silent=False):
119
+ def _std_copy(
120
+ self,
121
+ source,
122
+ destination,
123
+ smartcp_threshold=0,
124
+ intent="in",
125
+ pack=False,
126
+ silent=False,
127
+ ):
108
128
  """Extended copy for (possibly) multi GRIB file."""
109
129
  # Might be multipart
110
130
  if self.is_xgrib(source):
@@ -112,22 +132,38 @@ class GRIB_Tool(addons.FtrawEnableAddon):
112
132
  if isinstance(destination, str) and not pack:
113
133
  with self.sh.mute_stderr():
114
134
  idx = self._std_grib_index_get(source)
115
- destdir = self.sh.path.abspath(self.sh.path.expanduser(destination) + ".d")
135
+ destdir = self.sh.path.abspath(
136
+ self.sh.path.expanduser(destination) + ".d"
137
+ )
116
138
  rc = rc and self.sh.mkdir(destdir)
117
139
  target_idx = list()
118
- for (i, a_mpart) in enumerate(idx):
119
- target_idx.append(self.sh.path.join(destdir, 'GRIB_mpart{:06d}'.format(i)))
120
- rc = rc and self._backend_cp(a_mpart, target_idx[-1],
121
- smartcp_threshold=smartcp_threshold, intent=intent)
122
- rc = rc and self._std_grib_index_write(destination, target_idx)
123
- if intent == 'in':
140
+ for i, a_mpart in enumerate(idx):
141
+ target_idx.append(
142
+ self.sh.path.join(
143
+ destdir, "GRIB_mpart{:06d}".format(i)
144
+ )
145
+ )
146
+ rc = rc and self._backend_cp(
147
+ a_mpart,
148
+ target_idx[-1],
149
+ smartcp_threshold=smartcp_threshold,
150
+ intent=intent,
151
+ )
152
+ rc = rc and self._std_grib_index_write(
153
+ destination, target_idx
154
+ )
155
+ if intent == "in":
124
156
  self.sh.chmod(destination, 0o444)
125
157
  else:
126
158
  rc = rc and self.xgrib_pack(source, destination)
127
159
  else:
128
160
  # Usual file or file descriptor
129
- rc = self._backend_cp(source, destination,
130
- smartcp_threshold=smartcp_threshold, intent=intent)
161
+ rc = self._backend_cp(
162
+ source,
163
+ destination,
164
+ smartcp_threshold=smartcp_threshold,
165
+ intent=intent,
166
+ )
131
167
  return rc
132
168
 
133
169
  grib_cp = grib_copy = _std_copy
@@ -136,7 +172,7 @@ class GRIB_Tool(addons.FtrawEnableAddon):
136
172
  """Extended mv for (possibly) multi GRIB file."""
137
173
  # Might be multipart
138
174
  if self.is_xgrib(source):
139
- intent = 'inout' if self.sh.access(source, self.sh.W_OK) else 'in'
175
+ intent = "inout" if self.sh.access(source, self.sh.W_OK) else "in"
140
176
  rc = self._std_copy(source, destination, intent=intent)
141
177
  rc = rc and self._std_remove(source)
142
178
  else:
@@ -146,7 +182,9 @@ class GRIB_Tool(addons.FtrawEnableAddon):
146
182
  grib_mv = grib_move = _std_move
147
183
 
148
184
  def _pack_stream(self, source, stdout=True):
149
- cmd = ['cat', ]
185
+ cmd = [
186
+ "cat",
187
+ ]
150
188
  cmd.extend(self._std_grib_index_get(source))
151
189
  return self.sh.popen(cmd, stdout=stdout, bufsize=8192)
152
190
 
@@ -159,14 +197,14 @@ class GRIB_Tool(addons.FtrawEnableAddon):
159
197
  total += size
160
198
  return total
161
199
 
162
- def xgrib_pack(self, source, destination, intent='in'):
200
+ def xgrib_pack(self, source, destination, intent="in"):
163
201
  """Manually pack a multi GRIB."""
164
202
  if isinstance(destination, str):
165
203
  tmpfile = self.sh.safe_fileaddsuffix(destination)
166
- with open(tmpfile, 'wb') as fd:
204
+ with open(tmpfile, "wb") as fd:
167
205
  p = self._pack_stream(source, stdout=fd)
168
206
  self.sh.pclose(p)
169
- if intent == 'in':
207
+ if intent == "in":
170
208
  self.sh.chmod(tmpfile, 0o444)
171
209
  return self.sh.move(tmpfile, destination)
172
210
  else:
@@ -177,13 +215,16 @@ class GRIB_Tool(addons.FtrawEnableAddon):
177
215
  def _std_forcepack(self, source, destination=None):
178
216
  """Returned a path to a packed data."""
179
217
  if self.is_xgrib(source):
180
- destination = (destination if destination else
181
- self.sh.safe_fileaddsuffix(source))
218
+ destination = (
219
+ destination
220
+ if destination
221
+ else self.sh.safe_fileaddsuffix(source)
222
+ )
182
223
  if not self.sh.path.exists(destination):
183
224
  if self.xgrib_pack(source, destination):
184
225
  return destination
185
226
  else:
186
- raise OSError('XGrib packing failed')
227
+ raise OSError("XGrib packing failed")
187
228
  else:
188
229
  return destination
189
230
  else:
@@ -191,8 +232,16 @@ class GRIB_Tool(addons.FtrawEnableAddon):
191
232
 
192
233
  grib_forcepack = _std_forcepack
193
234
 
194
- def _std_ftput(self, source, destination, hostname=None, logname=None,
195
- port=DEFAULT_FTP_PORT, cpipeline=None, sync=False):
235
+ def _std_ftput(
236
+ self,
237
+ source,
238
+ destination,
239
+ hostname=None,
240
+ logname=None,
241
+ port=DEFAULT_FTP_PORT,
242
+ cpipeline=None,
243
+ sync=False,
244
+ ):
196
245
  """On the fly packing and ftp."""
197
246
  if self.is_xgrib(source):
198
247
  if cpipeline is not None:
@@ -202,19 +251,35 @@ class GRIB_Tool(addons.FtrawEnableAddon):
202
251
  if ftp:
203
252
  packed_size = self._packed_size(source)
204
253
  p = self._pack_stream(source)
205
- rc = ftp.put(p.stdout, destination, size=packed_size, exact=True)
254
+ rc = ftp.put(
255
+ p.stdout, destination, size=packed_size, exact=True
256
+ )
206
257
  self.sh.pclose(p)
207
258
  ftp.close()
208
259
  else:
209
260
  rc = False
210
261
  return rc
211
262
  else:
212
- return self.sh.ftput(source, destination, hostname=hostname,
213
- logname=logname, port=port,
214
- cpipeline=cpipeline, sync=sync)
263
+ return self.sh.ftput(
264
+ source,
265
+ destination,
266
+ hostname=hostname,
267
+ logname=logname,
268
+ port=port,
269
+ cpipeline=cpipeline,
270
+ sync=sync,
271
+ )
215
272
 
216
- def _std_rawftput(self, source, destination, hostname=None, logname=None,
217
- port=None, cpipeline=None, sync=False):
273
+ def _std_rawftput(
274
+ self,
275
+ source,
276
+ destination,
277
+ hostname=None,
278
+ logname=None,
279
+ port=None,
280
+ cpipeline=None,
281
+ sync=False,
282
+ ):
218
283
  """Use ftserv as much as possible."""
219
284
  if self.is_xgrib(source):
220
285
  if cpipeline is not None:
@@ -222,52 +287,82 @@ class GRIB_Tool(addons.FtrawEnableAddon):
222
287
  if self.sh.ftraw and self.rawftshell is not None:
223
288
  # Copy the GRIB pieces individually
224
289
  pieces = self.xgrib_index_get(source)
225
- newsources = [str(self.sh.copy2ftspool(piece)) for piece in pieces]
226
- request = newsources[0] + '.request'
227
- with open(request, 'w') as request_fh:
228
- request_fh.writelines('\n'.join(newsources))
290
+ newsources = [
291
+ str(self.sh.copy2ftspool(piece)) for piece in pieces
292
+ ]
293
+ request = newsources[0] + ".request"
294
+ with open(request, "w") as request_fh:
295
+ request_fh.writelines("\n".join(newsources))
229
296
  self.sh.readonly(request)
230
- rc = self.sh.ftserv_put(request, destination,
231
- hostname=hostname, logname=logname, port=port,
232
- specialshell=self.rawftshell, sync=sync)
297
+ rc = self.sh.ftserv_put(
298
+ request,
299
+ destination,
300
+ hostname=hostname,
301
+ logname=logname,
302
+ port=port,
303
+ specialshell=self.rawftshell,
304
+ sync=sync,
305
+ )
233
306
  self.sh.rm(request)
234
307
  return rc
235
308
  else:
236
309
  if port is None:
237
310
  port = DEFAULT_FTP_PORT
238
- return self._std_ftput(source, destination,
239
- hostname=hostname, logname=logname,
240
- port=port, sync=sync)
311
+ return self._std_ftput(
312
+ source,
313
+ destination,
314
+ hostname=hostname,
315
+ logname=logname,
316
+ port=port,
317
+ sync=sync,
318
+ )
241
319
  else:
242
- return self.sh.rawftput(source, destination, hostname=hostname,
243
- logname=logname, port=port,
244
- cpipeline=cpipeline, sync=sync)
320
+ return self.sh.rawftput(
321
+ source,
322
+ destination,
323
+ hostname=hostname,
324
+ logname=logname,
325
+ port=port,
326
+ cpipeline=cpipeline,
327
+ sync=sync,
328
+ )
245
329
 
246
330
  grib_ftput = _std_ftput
247
331
  grib_rawftput = _std_rawftput
248
332
 
249
- def _std_scpput(self, source, destination, hostname, logname=None, cpipeline=None):
333
+ def _std_scpput(
334
+ self, source, destination, hostname, logname=None, cpipeline=None
335
+ ):
250
336
  """On the fly packing and scp."""
251
337
  if self.is_xgrib(source):
252
338
  if cpipeline is not None:
253
339
  raise OSError("It's not allowed to compress xgrib files.")
254
- logname = self.sh.fix_ftuser(hostname, logname, fatal=False, defaults_to_user=False)
340
+ logname = self.sh.fix_ftuser(
341
+ hostname, logname, fatal=False, defaults_to_user=False
342
+ )
255
343
  ssh = self.sh.ssh(hostname, logname)
256
344
  permissions = ssh.get_permissions(source)
257
345
  # remove the .d companion directory (scp_stream removes the destination)
258
346
  # go on on failure : the .d lingers on, but the grib will be self-contained
259
- ssh.remove(destination + '.d')
347
+ ssh.remove(destination + ".d")
260
348
  p = self._pack_stream(source)
261
- rc = ssh.scpput_stream(p.stdout, destination, permissions=permissions)
349
+ rc = ssh.scpput_stream(
350
+ p.stdout, destination, permissions=permissions
351
+ )
262
352
  self.sh.pclose(p)
263
353
  return rc
264
354
  else:
265
- return self.sh.scpput(source, destination, hostname,
266
- logname=logname, cpipeline=cpipeline)
355
+ return self.sh.scpput(
356
+ source,
357
+ destination,
358
+ hostname,
359
+ logname=logname,
360
+ cpipeline=cpipeline,
361
+ )
267
362
 
268
363
  grib_scpput = _std_scpput
269
364
 
270
- @addons.require_external_addon('ecfs')
365
+ @addons.require_external_addon("ecfs")
271
366
  def grib_ecfsput(self, source, target, cpipeline=None, options=None):
272
367
  """Put a grib resource using ECfs.
273
368
 
@@ -282,25 +377,33 @@ class GRIB_Tool(addons.FtrawEnableAddon):
282
377
  raise OSError("It's not allowed to compress xgrib files.")
283
378
  psource = self.sh.safe_fileaddsuffix(source)
284
379
  try:
285
- rc = self.xgrib_pack(source=source,
286
- destination=psource)
380
+ rc = self.xgrib_pack(source=source, destination=psource)
287
381
  dict_args = dict()
288
382
  if rc:
289
- rc, dict_args = self.sh.ecfsput(source=psource,
290
- target=target,
291
- options=options)
383
+ rc, dict_args = self.sh.ecfsput(
384
+ source=psource, target=target, options=options
385
+ )
292
386
  finally:
293
387
  self.sh.rm(psource)
294
388
  return rc, dict_args
295
389
  else:
296
- return self.sh.ecfsput(source=source,
297
- target=target,
298
- options=options,
299
- cpipeline=cpipeline)
300
-
301
- @addons.require_external_addon('ectrans')
302
- def grib_ectransput(self, source, target, gateway=None, remote=None,
303
- cpipeline=None, sync=False):
390
+ return self.sh.ecfsput(
391
+ source=source,
392
+ target=target,
393
+ options=options,
394
+ cpipeline=cpipeline,
395
+ )
396
+
397
+ @addons.require_external_addon("ectrans")
398
+ def grib_ectransput(
399
+ self,
400
+ source,
401
+ target,
402
+ gateway=None,
403
+ remote=None,
404
+ cpipeline=None,
405
+ sync=False,
406
+ ):
304
407
  """Put a grib resource using ECtrans.
305
408
 
306
409
  :param source: source file
@@ -316,25 +419,28 @@ class GRIB_Tool(addons.FtrawEnableAddon):
316
419
  raise OSError("It's not allowed to compress xgrib files.")
317
420
  psource = self.sh.safe_fileaddsuffix(source)
318
421
  try:
319
- rc = self.xgrib_pack(source=source,
320
- destination=psource)
422
+ rc = self.xgrib_pack(source=source, destination=psource)
321
423
  dict_args = dict()
322
424
  if rc:
323
- rc, dict_args = self.sh.raw_ectransput(source=psource,
324
- target=target,
325
- gateway=gateway,
326
- remote=remote,
327
- sync=sync)
425
+ rc, dict_args = self.sh.raw_ectransput(
426
+ source=psource,
427
+ target=target,
428
+ gateway=gateway,
429
+ remote=remote,
430
+ sync=sync,
431
+ )
328
432
  finally:
329
433
  self.sh.rm(psource)
330
434
  return rc, dict_args
331
435
  else:
332
- return self.sh.ectransput(source=source,
333
- target=target,
334
- gateway=gateway,
335
- remote=remote,
336
- cpipeline=cpipeline,
337
- sync=sync)
436
+ return self.sh.ectransput(
437
+ source=source,
438
+ target=target,
439
+ gateway=gateway,
440
+ remote=remote,
441
+ cpipeline=cpipeline,
442
+ sync=sync,
443
+ )
338
444
 
339
445
 
340
446
  @algo_component_deco_mixin_autodoc
@@ -356,23 +462,37 @@ class EcGribDecoMixin(AlgoComponentDecoMixin):
356
462
  gribapi_lib = None
357
463
  if rh is not None:
358
464
  if not isinstance(rh, (list, tuple)):
359
- rh = [rh, ]
465
+ rh = [
466
+ rh,
467
+ ]
360
468
  for a_rh in rh:
361
469
  libs = self.system.ldd(a_rh.container.localpath())
362
470
  a_eccodes_lib = None
363
471
  a_gribapi_lib = None
364
472
  for lib, path in libs.items():
365
- if re.match(r'^libeccodes(?:-[.0-9]+)?\.so(?:\.[.0-9]+)?$', lib):
473
+ if re.match(
474
+ r"^libeccodes(?:-[.0-9]+)?\.so(?:\.[.0-9]+)?$", lib
475
+ ):
366
476
  a_eccodes_lib = path
367
- if re.match(r'^libgrib_api(?:-[.0-9]+)?\.so(?:\.[.0-9]+)?$', lib):
477
+ if re.match(
478
+ r"^libgrib_api(?:-[.0-9]+)?\.so(?:\.[.0-9]+)?$", lib
479
+ ):
368
480
  a_gribapi_lib = path
369
481
  if a_eccodes_lib:
370
- self.algoassert(eccodes_lib is None or (eccodes_lib == a_eccodes_lib),
371
- "ecCodes library inconsistency (rh: {!s})".format(a_rh))
482
+ self.algoassert(
483
+ eccodes_lib is None or (eccodes_lib == a_eccodes_lib),
484
+ "ecCodes library inconsistency (rh: {!s})".format(
485
+ a_rh
486
+ ),
487
+ )
372
488
  eccodes_lib = a_eccodes_lib
373
489
  if a_gribapi_lib:
374
- self.algoassert(gribapi_lib is None or (gribapi_lib == a_gribapi_lib),
375
- "grib_api library inconsistency (rh: {!s})".format(a_rh))
490
+ self.algoassert(
491
+ gribapi_lib is None or (gribapi_lib == a_gribapi_lib),
492
+ "grib_api library inconsistency (rh: {!s})".format(
493
+ a_rh
494
+ ),
495
+ )
376
496
  gribapi_lib = a_gribapi_lib
377
497
  return eccodes_lib, gribapi_lib
378
498
 
@@ -380,71 +500,99 @@ class EcGribDecoMixin(AlgoComponentDecoMixin):
380
500
  """Add axtra definitions/samples to the library path."""
381
501
  for gdef in self.context.sequence.effective_inputs(role=a_role):
382
502
  local_path = gdef.rh.container.localpath()
383
- new_path = (local_path if self.system.path.isdir(local_path)
384
- else self.system.path.dirname(local_path))
503
+ new_path = (
504
+ local_path
505
+ if self.system.path.isdir(local_path)
506
+ else self.system.path.dirname(local_path)
507
+ )
385
508
  # NB: Grib-API doesn't understand relative paths...
386
509
  new_path = self.system.path.abspath(new_path)
387
510
  self.env.setgenericpath(a_var, new_path, pos=0)
388
511
 
389
512
  def _gribapi_envsetup(self, gribapi_lib):
390
513
  """Setup environment variables for grib_api."""
391
- defvar = 'GRIB_DEFINITION_PATH'
392
- samplevar = 'GRIB_SAMPLES_PATH'
514
+ defvar = "GRIB_DEFINITION_PATH"
515
+ samplevar = "GRIB_SAMPLES_PATH"
393
516
  if gribapi_lib is not None:
394
517
  gribapi_root = self.system.path.dirname(gribapi_lib)
395
518
  gribapi_root = self.system.path.split(gribapi_root)[0]
396
- gribapi_share = self.system.path.join(gribapi_root, 'share', 'grib_api')
519
+ gribapi_share = self.system.path.join(
520
+ gribapi_root, "share", "grib_api"
521
+ )
397
522
  if defvar not in self.env:
398
523
  # This one is for compatibility with old versions of the gribapi !
399
- self.env.setgenericpath(defvar,
400
- self.system.path.join(gribapi_root, 'share', 'definitions'))
524
+ self.env.setgenericpath(
525
+ defvar,
526
+ self.system.path.join(
527
+ gribapi_root, "share", "definitions"
528
+ ),
529
+ )
401
530
  # This should be the lastest one:
402
- self.env.setgenericpath(defvar,
403
- self.system.path.join(gribapi_share, 'definitions'))
531
+ self.env.setgenericpath(
532
+ defvar, self.system.path.join(gribapi_share, "definitions")
533
+ )
404
534
  if samplevar not in self.env:
405
535
  # This one is for compatibility with old versions of the gribapi !
406
- self.env.setgenericpath(samplevar,
407
- self.system.path.join(gribapi_root, 'ifs_samples', 'grib1'))
536
+ self.env.setgenericpath(
537
+ samplevar,
538
+ self.system.path.join(
539
+ gribapi_root, "ifs_samples", "grib1"
540
+ ),
541
+ )
408
542
  # This should be the lastest one:
409
- self.env.setgenericpath(samplevar,
410
- self.system.path.join(gribapi_share, 'ifs_samples', 'grib1'))
543
+ self.env.setgenericpath(
544
+ samplevar,
545
+ self.system.path.join(
546
+ gribapi_share, "ifs_samples", "grib1"
547
+ ),
548
+ )
411
549
  else:
412
550
  # Use the default GRIB-API config if the ldd approach fails
413
- self.export('gribapi')
551
+ self.export("gribapi")
414
552
  return defvar, samplevar
415
553
 
416
554
  def gribapi_setup(self, rh, opts):
417
555
  """Setup the grib_api related stuff."""
418
556
  _, gribapi_lib = self._ecgrib_libs_detext(rh)
419
557
  defvar, samplevar = self._gribapi_envsetup(gribapi_lib)
420
- self._ecgrib_additional_config('AdditionalGribAPIDefinitions', defvar)
421
- self._ecgrib_additional_config('AdditionalGribAPISamples', samplevar)
558
+ self._ecgrib_additional_config("AdditionalGribAPIDefinitions", defvar)
559
+ self._ecgrib_additional_config("AdditionalGribAPISamples", samplevar)
422
560
  # Recap
423
561
  for a_var in (defvar, samplevar):
424
- logger.info('After gribapi_setup %s = %s', a_var, self.env.getvar(a_var))
562
+ logger.info(
563
+ "After gribapi_setup %s = %s", a_var, self.env.getvar(a_var)
564
+ )
425
565
 
426
566
  def _eccodes_envsetup(self, eccodes_lib):
427
567
  """Setup environment variables for ecCodes."""
428
- dep_warn = ("%s is left unconfigured because the old grib_api's variable is defined." +
429
- "Please remove that !")
568
+ dep_warn = (
569
+ "%s is left unconfigured because the old grib_api's variable is defined."
570
+ + "Please remove that !"
571
+ )
430
572
  eccodes_root = self.system.path.dirname(eccodes_lib)
431
573
  eccodes_root = self.system.path.split(eccodes_root)[0]
432
- eccodes_share = self.system.path.join(eccodes_root, 'share', 'eccodes')
433
- defvar = 'ECCODES_DEFINITION_PATH'
574
+ eccodes_share = self.system.path.join(eccodes_root, "share", "eccodes")
575
+ defvar = "ECCODES_DEFINITION_PATH"
434
576
  if defvar not in self.env:
435
- if 'GRIB_DEFINITION_PATH' in self.env:
577
+ if "GRIB_DEFINITION_PATH" in self.env:
436
578
  logger.warning(dep_warn, defvar)
437
- defvar = 'GRIB_DEFINITION_PATH'
579
+ defvar = "GRIB_DEFINITION_PATH"
438
580
  else:
439
- self.env.setgenericpath(defvar, self.system.path.join(eccodes_share, 'definitions'))
440
- samplevar = 'ECCODES_SAMPLES_PATH'
581
+ self.env.setgenericpath(
582
+ defvar, self.system.path.join(eccodes_share, "definitions")
583
+ )
584
+ samplevar = "ECCODES_SAMPLES_PATH"
441
585
  if samplevar not in self.env:
442
- if 'GRIB_SAMPLES_PATH' in self.env:
586
+ if "GRIB_SAMPLES_PATH" in self.env:
443
587
  logger.warning(dep_warn, samplevar)
444
- samplevar = 'GRIB_SAMPLES_PATH'
588
+ samplevar = "GRIB_SAMPLES_PATH"
445
589
  else:
446
- self.env.setgenericpath(samplevar,
447
- self.system.path.join(eccodes_share, 'ifs_samples', 'grib1'))
590
+ self.env.setgenericpath(
591
+ samplevar,
592
+ self.system.path.join(
593
+ eccodes_share, "ifs_samples", "grib1"
594
+ ),
595
+ )
448
596
  return defvar, samplevar
449
597
 
450
598
  def eccodes_setup(self, rh, opts, compat=False, fatal=True):
@@ -462,26 +610,38 @@ class EcGribDecoMixin(AlgoComponentDecoMixin):
462
610
  defvar, samplevar = self._gribapi_envsetup(gribapi_lib)
463
611
  else:
464
612
  if fatal:
465
- raise RuntimeError('No suitable configuration found for ecCodes.')
613
+ raise RuntimeError(
614
+ "No suitable configuration found for ecCodes."
615
+ )
466
616
  else:
467
617
  logger.error("ecCodes was not found !")
468
618
  return
469
619
  # Then, inspect the context to look for customised search paths
470
- self._ecgrib_additional_config(('AdditionalGribAPIDefinitions', 'AdditionalEcCodesDefinitions'),
471
- defvar)
472
- self._ecgrib_additional_config(('AdditionalGribAPISamples', 'AdditionalEcCodesSamples'),
473
- samplevar)
620
+ self._ecgrib_additional_config(
621
+ ("AdditionalGribAPIDefinitions", "AdditionalEcCodesDefinitions"),
622
+ defvar,
623
+ )
624
+ self._ecgrib_additional_config(
625
+ ("AdditionalGribAPISamples", "AdditionalEcCodesSamples"), samplevar
626
+ )
474
627
  # Recap
475
628
  for a_var in (defvar, samplevar):
476
- logger.info('After eccodes_setup (compat=%s) : %s = %s', str(compat),
477
- a_var, self.env.getvar(a_var))
629
+ logger.info(
630
+ "After eccodes_setup (compat=%s) : %s = %s",
631
+ str(compat),
632
+ a_var,
633
+ self.env.getvar(a_var),
634
+ )
478
635
 
479
636
  def _ecgrib_mixin_setup(self, rh, opts):
480
- self.eccodes_setup(rh, opts,
481
- compat=self._ECGRIB_SETUP_COMPAT,
482
- fatal=self._ECGRIB_SETUP_FATAL)
637
+ self.eccodes_setup(
638
+ rh,
639
+ opts,
640
+ compat=self._ECGRIB_SETUP_COMPAT,
641
+ fatal=self._ECGRIB_SETUP_FATAL,
642
+ )
483
643
 
484
- _MIXIN_PREPARE_HOOKS = (_ecgrib_mixin_setup, )
644
+ _MIXIN_PREPARE_HOOKS = (_ecgrib_mixin_setup,)
485
645
 
486
646
 
487
647
  class GRIBAPI_Tool(addons.Addon):
@@ -490,12 +650,12 @@ class GRIBAPI_Tool(addons.Addon):
490
650
  """
491
651
 
492
652
  _footprint = dict(
493
- info = 'Default GRIBAPI system interface',
494
- attr = dict(
495
- kind = dict(
496
- values = ['gribapi'],
653
+ info="Default GRIBAPI system interface",
654
+ attr=dict(
655
+ kind=dict(
656
+ values=["gribapi"],
497
657
  ),
498
- )
658
+ ),
499
659
  )
500
660
 
501
661
  def __init__(self, *args, **kw):
@@ -513,17 +673,19 @@ class GRIBAPI_Tool(addons.Addon):
513
673
 
514
674
  def _spawn_wrap(self, cmd, **kw):
515
675
  """Internal method calling standard shell spawn."""
516
- cmd[0] = 'bin' + self.sh.path.sep + cmd[0]
676
+ cmd[0] = "bin" + self.sh.path.sep + cmd[0]
517
677
  return super()._spawn_wrap(cmd, **kw)
518
678
 
519
679
  def _actual_diff(self, grib1, grib2, skipkeys, **kw):
520
680
  """Run the actual GRIBAPI command."""
521
- cmd = ['grib_compare', '-r', '-b', ','.join(skipkeys), grib1, grib2]
522
- kw['fatal'] = False
523
- kw['output'] = False
681
+ cmd = ["grib_compare", "-r", "-b", ",".join(skipkeys), grib1, grib2]
682
+ kw["fatal"] = False
683
+ kw["output"] = False
524
684
  return self._spawn_wrap(cmd, **kw)
525
685
 
526
- def grib_diff(self, grib1, grib2, skipkeys=('generatingProcessIdentifier',), **kw):
686
+ def grib_diff(
687
+ self, grib1, grib2, skipkeys=("generatingProcessIdentifier",), **kw
688
+ ):
527
689
  """
528
690
  Difference between two GRIB files (using the GRIB-API)
529
691
 
@@ -538,15 +700,15 @@ class GRIBAPI_Tool(addons.Addon):
538
700
  """
539
701
 
540
702
  # Are multipart GRIB suported ?
541
- xgrib_support = 'grib' in self.sh.loaded_addons()
703
+ xgrib_support = "grib" in self.sh.loaded_addons()
542
704
  grib1_ori = grib1
543
705
  grib2_ori = grib2
544
706
  if xgrib_support:
545
707
  if self.sh.is_xgrib(grib1):
546
- grib1 = self.sh.safe_fileaddsuffix(grib1_ori) + '_diffcat'
708
+ grib1 = self.sh.safe_fileaddsuffix(grib1_ori) + "_diffcat"
547
709
  self.sh.xgrib_pack(grib1_ori, grib1)
548
710
  if self.sh.is_xgrib(grib2):
549
- grib2 = self.sh.safe_fileaddsuffix(grib2_ori) + '_diffcat'
711
+ grib2 = self.sh.safe_fileaddsuffix(grib2_ori) + "_diffcat"
550
712
  self.sh.xgrib_pack(grib2_ori, grib2)
551
713
 
552
714
  rc = self._actual_diff(grib1, grib2, skipkeys, **kw)