vortex-nwp 2.0.0b1__py3-none-any.whl → 2.0.0b2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (139) hide show
  1. vortex/__init__.py +59 -45
  2. vortex/algo/__init__.py +3 -2
  3. vortex/algo/components.py +940 -614
  4. vortex/algo/mpitools.py +802 -497
  5. vortex/algo/serversynctools.py +34 -33
  6. vortex/config.py +19 -22
  7. vortex/data/__init__.py +9 -3
  8. vortex/data/abstractstores.py +593 -655
  9. vortex/data/containers.py +217 -162
  10. vortex/data/contents.py +65 -39
  11. vortex/data/executables.py +93 -102
  12. vortex/data/flow.py +40 -34
  13. vortex/data/geometries.py +228 -132
  14. vortex/data/handlers.py +428 -225
  15. vortex/data/outflow.py +15 -15
  16. vortex/data/providers.py +185 -163
  17. vortex/data/resources.py +48 -42
  18. vortex/data/stores.py +544 -413
  19. vortex/gloves.py +114 -87
  20. vortex/layout/__init__.py +1 -8
  21. vortex/layout/contexts.py +150 -84
  22. vortex/layout/dataflow.py +353 -202
  23. vortex/layout/monitor.py +264 -128
  24. vortex/nwp/__init__.py +5 -2
  25. vortex/nwp/algo/__init__.py +14 -5
  26. vortex/nwp/algo/assim.py +205 -151
  27. vortex/nwp/algo/clim.py +683 -517
  28. vortex/nwp/algo/coupling.py +447 -225
  29. vortex/nwp/algo/eda.py +437 -229
  30. vortex/nwp/algo/eps.py +403 -231
  31. vortex/nwp/algo/forecasts.py +420 -271
  32. vortex/nwp/algo/fpserver.py +683 -307
  33. vortex/nwp/algo/ifsnaming.py +205 -145
  34. vortex/nwp/algo/ifsroot.py +210 -122
  35. vortex/nwp/algo/monitoring.py +132 -76
  36. vortex/nwp/algo/mpitools.py +321 -191
  37. vortex/nwp/algo/odbtools.py +617 -353
  38. vortex/nwp/algo/oopsroot.py +449 -273
  39. vortex/nwp/algo/oopstests.py +90 -56
  40. vortex/nwp/algo/request.py +287 -206
  41. vortex/nwp/algo/stdpost.py +878 -522
  42. vortex/nwp/data/__init__.py +22 -4
  43. vortex/nwp/data/assim.py +125 -137
  44. vortex/nwp/data/boundaries.py +121 -68
  45. vortex/nwp/data/climfiles.py +193 -211
  46. vortex/nwp/data/configfiles.py +73 -69
  47. vortex/nwp/data/consts.py +426 -401
  48. vortex/nwp/data/ctpini.py +59 -43
  49. vortex/nwp/data/diagnostics.py +94 -66
  50. vortex/nwp/data/eda.py +50 -51
  51. vortex/nwp/data/eps.py +195 -146
  52. vortex/nwp/data/executables.py +440 -434
  53. vortex/nwp/data/fields.py +63 -48
  54. vortex/nwp/data/gridfiles.py +183 -111
  55. vortex/nwp/data/logs.py +250 -217
  56. vortex/nwp/data/modelstates.py +180 -151
  57. vortex/nwp/data/monitoring.py +72 -99
  58. vortex/nwp/data/namelists.py +254 -202
  59. vortex/nwp/data/obs.py +400 -308
  60. vortex/nwp/data/oopsexec.py +22 -20
  61. vortex/nwp/data/providers.py +90 -65
  62. vortex/nwp/data/query.py +71 -82
  63. vortex/nwp/data/stores.py +49 -36
  64. vortex/nwp/data/surfex.py +136 -137
  65. vortex/nwp/syntax/__init__.py +1 -1
  66. vortex/nwp/syntax/stdattrs.py +173 -111
  67. vortex/nwp/tools/__init__.py +2 -2
  68. vortex/nwp/tools/addons.py +22 -17
  69. vortex/nwp/tools/agt.py +24 -12
  70. vortex/nwp/tools/bdap.py +16 -5
  71. vortex/nwp/tools/bdcp.py +4 -1
  72. vortex/nwp/tools/bdm.py +3 -0
  73. vortex/nwp/tools/bdmp.py +14 -9
  74. vortex/nwp/tools/conftools.py +728 -378
  75. vortex/nwp/tools/drhook.py +12 -8
  76. vortex/nwp/tools/grib.py +65 -39
  77. vortex/nwp/tools/gribdiff.py +22 -17
  78. vortex/nwp/tools/ifstools.py +82 -42
  79. vortex/nwp/tools/igastuff.py +167 -143
  80. vortex/nwp/tools/mars.py +14 -2
  81. vortex/nwp/tools/odb.py +234 -125
  82. vortex/nwp/tools/partitioning.py +61 -37
  83. vortex/nwp/tools/satrad.py +27 -12
  84. vortex/nwp/util/async.py +83 -55
  85. vortex/nwp/util/beacon.py +10 -10
  86. vortex/nwp/util/diffpygram.py +174 -86
  87. vortex/nwp/util/ens.py +144 -63
  88. vortex/nwp/util/hooks.py +30 -19
  89. vortex/nwp/util/taskdeco.py +28 -24
  90. vortex/nwp/util/usepygram.py +278 -172
  91. vortex/nwp/util/usetnt.py +31 -17
  92. vortex/sessions.py +72 -39
  93. vortex/syntax/__init__.py +1 -1
  94. vortex/syntax/stdattrs.py +410 -171
  95. vortex/syntax/stddeco.py +31 -22
  96. vortex/toolbox.py +327 -192
  97. vortex/tools/__init__.py +11 -2
  98. vortex/tools/actions.py +125 -59
  99. vortex/tools/addons.py +111 -92
  100. vortex/tools/arm.py +42 -22
  101. vortex/tools/compression.py +72 -69
  102. vortex/tools/date.py +11 -4
  103. vortex/tools/delayedactions.py +242 -132
  104. vortex/tools/env.py +75 -47
  105. vortex/tools/folder.py +342 -171
  106. vortex/tools/grib.py +311 -149
  107. vortex/tools/lfi.py +423 -216
  108. vortex/tools/listings.py +109 -40
  109. vortex/tools/names.py +218 -156
  110. vortex/tools/net.py +632 -298
  111. vortex/tools/parallelism.py +93 -61
  112. vortex/tools/prestaging.py +55 -31
  113. vortex/tools/schedulers.py +172 -105
  114. vortex/tools/services.py +402 -333
  115. vortex/tools/storage.py +293 -358
  116. vortex/tools/surfex.py +24 -24
  117. vortex/tools/systems.py +1211 -631
  118. vortex/tools/targets.py +156 -100
  119. vortex/util/__init__.py +1 -1
  120. vortex/util/config.py +377 -327
  121. vortex/util/empty.py +2 -2
  122. vortex/util/helpers.py +56 -24
  123. vortex/util/introspection.py +18 -12
  124. vortex/util/iosponge.py +8 -4
  125. vortex/util/roles.py +4 -6
  126. vortex/util/storefunctions.py +39 -13
  127. vortex/util/structs.py +3 -3
  128. vortex/util/worker.py +29 -17
  129. vortex_nwp-2.0.0b2.dist-info/METADATA +66 -0
  130. vortex_nwp-2.0.0b2.dist-info/RECORD +142 -0
  131. {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.0.0b2.dist-info}/WHEEL +1 -1
  132. vortex/layout/appconf.py +0 -109
  133. vortex/layout/jobs.py +0 -1276
  134. vortex/layout/nodes.py +0 -1424
  135. vortex/layout/subjobs.py +0 -464
  136. vortex_nwp-2.0.0b1.dist-info/METADATA +0 -50
  137. vortex_nwp-2.0.0b1.dist-info/RECORD +0 -146
  138. {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.0.0b2.dist-info}/LICENSE +0 -0
  139. {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.0.0b2.dist-info}/top_level.txt +0 -0
vortex/tools/lfi.py CHANGED
@@ -29,7 +29,7 @@ logger = loggers.getLogger(__name__)
29
29
 
30
30
  def use_in_shell(sh, **kw):
31
31
  """Extend current shell with the LFI interface defined by optional arguments."""
32
- kw['shell'] = sh
32
+ kw["shell"] = sh
33
33
  return footprints.proxy.addon(**kw)
34
34
 
35
35
 
@@ -49,7 +49,9 @@ class LFI_Status:
49
49
  self._result = result or list()
50
50
 
51
51
  def __str__(self):
52
- return '{:s} | rc={:d} result={:d}>'.format(repr(self).rstrip('>'), self.rc, len(self.result))
52
+ return "{:s} | rc={:d} result={:d}>".format(
53
+ repr(self).rstrip(">"), self.rc, len(self.result)
54
+ )
53
55
 
54
56
  @property
55
57
  def ok(self):
@@ -107,42 +109,44 @@ class LFI_Tool_Raw(addons.FtrawEnableAddon):
107
109
  Interface to LFI commands through Perl wrappers.
108
110
  """
109
111
 
110
- LFI_HNDL_SPEC = ':1'
112
+ LFI_HNDL_SPEC = ":1"
111
113
  DR_HOOK_SILENT = 1
112
114
  DR_HOOK_NOT_MPI = 1
113
115
  DR_HOOK_ASSERT_MPI_INITIALIZED = 0
114
- OMP_STACKSIZE = '32M'
115
- KMP_STACKSIZE = '32M'
116
- KMP_MONITOR_STACKSIZE = '32M'
116
+ OMP_STACKSIZE = "32M"
117
+ KMP_STACKSIZE = "32M"
118
+ KMP_MONITOR_STACKSIZE = "32M"
117
119
 
118
120
  _footprint = dict(
119
- info = 'Default LFI system interface (Perl)',
120
- attr = dict(
121
- kind = dict(
122
- values = ['lfi'],
121
+ info="Default LFI system interface (Perl)",
122
+ attr=dict(
123
+ kind=dict(
124
+ values=["lfi"],
123
125
  ),
124
- cmd = dict(
125
- alias = ('lficmd',),
126
- default = 'lfitools',
126
+ cmd=dict(
127
+ alias=("lficmd",),
128
+ default="lfitools",
127
129
  ),
128
- path = dict(
129
- alias = ('lfipath',),
130
+ path=dict(
131
+ alias=("lfipath",),
130
132
  ),
131
- warnpack = dict(
132
- type = bool,
133
- optional = True,
134
- default = False,
133
+ warnpack=dict(
134
+ type=bool,
135
+ optional=True,
136
+ default=False,
135
137
  ),
136
- wraplanguage = dict(
137
- values = ['perl', ],
138
- default = 'perl',
139
- optional = True,
140
- doc_visibility = footprints.doc.visibility.ADVANCED,
138
+ wraplanguage=dict(
139
+ values=[
140
+ "perl",
141
+ ],
142
+ default="perl",
143
+ optional=True,
144
+ doc_visibility=footprints.doc.visibility.ADVANCED,
141
145
  ),
142
- toolkind = dict(
143
- default = 'lfitools',
146
+ toolkind=dict(
147
+ default="lfitools",
144
148
  ),
145
- )
149
+ ),
146
150
  )
147
151
 
148
152
  def __init__(self, *args, **kw):
@@ -154,7 +158,9 @@ class LFI_Tool_Raw(addons.FtrawEnableAddon):
154
158
  def lfitools_path(self):
155
159
  ctxtag = contexts.Context.tag_focus()
156
160
  if ctxtag not in self._lfitools_path:
157
- self._lfitools_path[ctxtag] = self.sh.path.join(self.actual_path, self.actual_cmd)
161
+ self._lfitools_path[ctxtag] = self.sh.path.join(
162
+ self.actual_path, self.actual_cmd
163
+ )
158
164
  self.sh.xperm(self._lfitools_path[ctxtag], force=True)
159
165
  return self._lfitools_path[ctxtag]
160
166
 
@@ -166,14 +172,20 @@ class LFI_Tool_Raw(addons.FtrawEnableAddon):
166
172
  def _spawn_wrap(self, func, cmd, **kw):
167
173
  """Tube to set LFITOOLS env variable."""
168
174
  self.env.LFITOOLS = self.lfitools_path
169
- return super()._spawn_wrap(['lfi_' + func, ] + cmd, **kw)
175
+ return super()._spawn_wrap(
176
+ [
177
+ "lfi_" + func,
178
+ ]
179
+ + cmd,
180
+ **kw,
181
+ )
170
182
 
171
183
  def is_xlfi(self, source):
172
184
  """Check if the given ``source`` is a multipart-lfi file."""
173
185
  rc = False
174
186
  if source and isinstance(source, str) and self.sh.path.exists(source):
175
- with open(source, 'rb') as fd:
176
- rc = fd.read(8) == b'LFI_ALTM'
187
+ with open(source, "rb") as fd:
188
+ rc = fd.read(8) == b"LFI_ALTM"
177
189
  return rc
178
190
 
179
191
  def _std_table(self, lfifile, **kw):
@@ -184,13 +196,13 @@ class LFI_Tool_Raw(addons.FtrawEnableAddon):
184
196
  * lfifile : lfi file name
185
197
 
186
198
  """
187
- cmd = ['lfilist', lfifile]
188
- kw['output'] = True
199
+ cmd = ["lfilist", lfifile]
200
+ kw["output"] = True
189
201
  rawout = self._spawn(cmd, **kw)
190
202
  return LFI_Status(
191
203
  rc=0,
192
204
  stdout=rawout,
193
- result=[tuple(eval(x)[0]) for x in rawout if x.startswith('[')]
205
+ result=[tuple(eval(x)[0]) for x in rawout if x.startswith("[")],
194
206
  )
195
207
 
196
208
  fa_table = lfi_table = _std_table
@@ -208,39 +220,39 @@ class LFI_Tool_Raw(addons.FtrawEnableAddon):
208
220
  * skipfields : LFI fields not to be compared
209
221
  * skiplength : Offset at which the comparison starts for each LFI fields
210
222
  """
211
- cmd = ['lfidiff', '--lfi-file-1', lfi1, '--lfi-file-2', lfi2]
223
+ cmd = ["lfidiff", "--lfi-file-1", lfi1, "--lfi-file-2", lfi2]
212
224
 
213
- maxprint = kw.pop('maxprint', 2)
225
+ maxprint = kw.pop("maxprint", 2)
214
226
  if maxprint:
215
- cmd.extend(['--max-print-diff', str(maxprint)])
227
+ cmd.extend(["--max-print-diff", str(maxprint)])
216
228
 
217
- skipfields = kw.pop('skipfields', 0)
229
+ skipfields = kw.pop("skipfields", 0)
218
230
  if skipfields:
219
- cmd.extend(['--lfi-skip-fields', str(skipfields)])
231
+ cmd.extend(["--lfi-skip-fields", str(skipfields)])
220
232
 
221
- skiplength = kw.pop('skiplength', 0)
233
+ skiplength = kw.pop("skiplength", 0)
222
234
  if skiplength:
223
- cmd.extend(['--lfi-skip-length', str(skiplength)])
235
+ cmd.extend(["--lfi-skip-length", str(skiplength)])
224
236
 
225
- kw['output'] = True
237
+ kw["output"] = True
226
238
 
227
239
  rawout = self._spawn(cmd, **kw)
228
- fields = [tuple(x.split(' ', 2)[-2:]) for x in rawout if re.match(r' (?:\!=|\+\+|\-\-)', x)]
240
+ fields = [
241
+ tuple(x.split(" ", 2)[-2:])
242
+ for x in rawout
243
+ if re.match(r" (?:\!=|\+\+|\-\-)", x)
244
+ ]
229
245
 
230
246
  trfields = Tracker(
231
- deleted=[x[1] for x in fields if x[0] == '--'],
232
- created=[x[1] for x in fields if x[0] == '++'],
233
- updated=[x[1] for x in fields if x[0] == '!='],
247
+ deleted=[x[1] for x in fields if x[0] == "--"],
248
+ created=[x[1] for x in fields if x[0] == "++"],
249
+ updated=[x[1] for x in fields if x[0] == "!="],
234
250
  )
235
251
 
236
252
  stlist = self.lfi_table(lfi1, output=True)
237
253
  trfields.unchanged = {x[0] for x in stlist.result} - set(trfields)
238
254
 
239
- return LFI_Status(
240
- rc=int(bool(fields)),
241
- stdout=rawout,
242
- result=trfields
243
- )
255
+ return LFI_Status(rc=int(bool(fields)), stdout=rawout, result=trfields)
244
256
 
245
257
  fa_diff = lfi_diff = _std_diff
246
258
 
@@ -253,17 +265,29 @@ class LFI_Tool_Raw(addons.FtrawEnableAddon):
253
265
  * fa2 : The new empty file
254
266
 
255
267
  """
256
- cmd = ['faempty', fa1, fa2]
268
+ cmd = ["faempty", fa1, fa2]
257
269
  self._spawn(cmd, **kw)
258
270
 
259
271
  def _pack_stream(self, source):
260
- return self._spawn_wrap('pack', [source, ],
261
- output=False,
262
- inpipe=True,
263
- bufsize=8192)
272
+ return self._spawn_wrap(
273
+ "pack",
274
+ [
275
+ source,
276
+ ],
277
+ output=False,
278
+ inpipe=True,
279
+ bufsize=8192,
280
+ )
264
281
 
265
282
  def _packed_size(self, source):
266
- out = self._spawn_wrap('size', [source, ], output=True, inpipe=False)
283
+ out = self._spawn_wrap(
284
+ "size",
285
+ [
286
+ source,
287
+ ],
288
+ output=True,
289
+ inpipe=False,
290
+ )
267
291
  try:
268
292
  return int(out[0])
269
293
  except ValueError:
@@ -273,14 +297,19 @@ class LFI_Tool_Raw(addons.FtrawEnableAddon):
273
297
  def _std_forcepack(self, source, destination=None):
274
298
  """Returned a path to a packed data."""
275
299
  if self.is_xlfi(source):
276
- destination = (destination if destination else
277
- self.sh.safe_fileaddsuffix(source))
300
+ destination = (
301
+ destination
302
+ if destination
303
+ else self.sh.safe_fileaddsuffix(source)
304
+ )
278
305
  if not self.sh.path.exists(destination):
279
- st = self._std_copy(source=source, destination=destination, pack=True)
306
+ st = self._std_copy(
307
+ source=source, destination=destination, pack=True
308
+ )
280
309
  if st:
281
310
  return destination
282
311
  else:
283
- raise OSError('XLFI packing failed')
312
+ raise OSError("XLFI packing failed")
284
313
  else:
285
314
  return destination
286
315
  else:
@@ -288,8 +317,16 @@ class LFI_Tool_Raw(addons.FtrawEnableAddon):
288
317
 
289
318
  fa_forcepack = lfi_forcepack = _std_forcepack
290
319
 
291
- def _std_ftput(self, source, destination, hostname=None, logname=None,
292
- port=DEFAULT_FTP_PORT, cpipeline=None, sync=False):
320
+ def _std_ftput(
321
+ self,
322
+ source,
323
+ destination,
324
+ hostname=None,
325
+ logname=None,
326
+ port=DEFAULT_FTP_PORT,
327
+ cpipeline=None,
328
+ sync=False,
329
+ ):
293
330
  """On the fly packing and ftp."""
294
331
  if self.is_xlfi(source):
295
332
  if cpipeline is not None:
@@ -301,64 +338,100 @@ class LFI_Tool_Raw(addons.FtrawEnableAddon):
301
338
  if ftp:
302
339
  packed_size = self._packed_size(source)
303
340
  p = self._pack_stream(source)
304
- st.rc = ftp.put(p.stdout, destination, size=packed_size, exact=True)
341
+ st.rc = ftp.put(
342
+ p.stdout, destination, size=packed_size, exact=True
343
+ )
305
344
  self.sh.pclose(p)
306
345
  st.result = [destination]
307
346
  st.stdout = [
308
- 'Connection time : {:f}'.format(ftp.length),
309
- 'Packed source size: {:d}'.format(packed_size),
310
- 'Actual target size: {:d}'.format(ftp.size(destination))
347
+ "Connection time : {:f}".format(ftp.length),
348
+ "Packed source size: {:d}".format(packed_size),
349
+ "Actual target size: {:d}".format(ftp.size(destination)),
311
350
  ]
312
351
  ftp.close()
313
352
  else:
314
353
  st.rc = 1
315
- st.result = ['Could not connect to ' + hostname + ' as user ' + logname]
354
+ st.result = [
355
+ "Could not connect to " + hostname + " as user " + logname
356
+ ]
316
357
  return st
317
358
  else:
318
- return self.sh.ftput(source, destination, hostname=hostname,
319
- logname=logname, cpipeline=cpipeline,
320
- port=port, sync=sync)
359
+ return self.sh.ftput(
360
+ source,
361
+ destination,
362
+ hostname=hostname,
363
+ logname=logname,
364
+ cpipeline=cpipeline,
365
+ port=port,
366
+ sync=sync,
367
+ )
321
368
 
322
- def _std_rawftput(self, source, destination, hostname=None, logname=None,
323
- port=None, cpipeline=None, sync=False):
369
+ def _std_rawftput(
370
+ self,
371
+ source,
372
+ destination,
373
+ hostname=None,
374
+ logname=None,
375
+ port=None,
376
+ cpipeline=None,
377
+ sync=False,
378
+ ):
324
379
  """Use ftserv as much as possible."""
325
380
  if self.is_xlfi(source):
326
381
  if cpipeline is not None:
327
382
  raise OSError("It's not allowed to compress xlfi files.")
328
383
  if self.sh.ftraw and self.rawftshell is not None:
329
- newsource = self.sh.copy2ftspool(source, fmt='lfi')
330
- rc = self.sh.ftserv_put(newsource, destination,
331
- hostname=hostname, logname=logname, port=port,
332
- specialshell=self.rawftshell, sync=sync)
384
+ newsource = self.sh.copy2ftspool(source, fmt="lfi")
385
+ rc = self.sh.ftserv_put(
386
+ newsource,
387
+ destination,
388
+ hostname=hostname,
389
+ logname=logname,
390
+ port=port,
391
+ specialshell=self.rawftshell,
392
+ sync=sync,
393
+ )
333
394
  self.sh.rm(newsource) # Delete the request file
334
395
  return rc
335
396
  else:
336
397
  if port is None:
337
398
  port = DEFAULT_FTP_PORT
338
- return self._std_ftput(source, destination, hostname, logname,
339
- port=port, sync=sync)
399
+ return self._std_ftput(
400
+ source,
401
+ destination,
402
+ hostname,
403
+ logname,
404
+ port=port,
405
+ sync=sync,
406
+ )
340
407
  else:
341
- return self.sh.rawftput(source, destination, hostname=hostname,
342
- logname=logname, port=port,
343
- cpipeline=cpipeline, sync=sync)
408
+ return self.sh.rawftput(
409
+ source,
410
+ destination,
411
+ hostname=hostname,
412
+ logname=logname,
413
+ port=port,
414
+ cpipeline=cpipeline,
415
+ sync=sync,
416
+ )
344
417
 
345
418
  fa_ftput = lfi_ftput = _std_ftput
346
419
  fa_rawftput = lfi_rawftput = _std_rawftput
347
420
 
348
- def _std_prepare(self, source, destination, intent='in'):
421
+ def _std_prepare(self, source, destination, intent="in"):
349
422
  """Check for the source and prepare the destination."""
350
- if intent not in ('in', 'inout'):
351
- raise ValueError('Incorrect value for intent ({})'.format(intent))
423
+ if intent not in ("in", "inout"):
424
+ raise ValueError("Incorrect value for intent ({})".format(intent))
352
425
  st = LFI_Status()
353
426
  if not self.sh.path.exists(source):
354
- logger.error('Missing source %s', source)
427
+ logger.error("Missing source %s", source)
355
428
  st.rc = 2
356
- st.stderr = 'No such source file or directory : [' + source + ']'
429
+ st.stderr = "No such source file or directory : [" + source + "]"
357
430
  return st
358
431
  if not self.sh.filecocoon(destination):
359
- raise OSError('Could not cocoon [' + destination + ']')
432
+ raise OSError("Could not cocoon [" + destination + "]")
360
433
  if not self.lfi_rm(destination):
361
- raise OSError('Could not clean destination [' + destination + ']')
434
+ raise OSError("Could not clean destination [" + destination + "]")
362
435
  return st
363
436
 
364
437
  def _std_remove(self, *args):
@@ -366,16 +439,30 @@ class LFI_Tool_Raw(addons.FtrawEnableAddon):
366
439
  st = LFI_Status(result=list())
367
440
  for pname in args:
368
441
  for objpath in self.sh.glob(pname):
369
- rc = self._spawn_wrap('remove', [objpath, ], output=False)
370
- st.result.append(dict(path=objpath,
371
- multi=self.is_xlfi(objpath), rc=rc))
442
+ rc = self._spawn_wrap(
443
+ "remove",
444
+ [
445
+ objpath,
446
+ ],
447
+ output=False,
448
+ )
449
+ st.result.append(
450
+ dict(path=objpath, multi=self.is_xlfi(objpath), rc=rc)
451
+ )
372
452
  st.rc = rc
373
453
  return st
374
454
 
375
455
  lfi_rm = lfi_remove = fa_rm = fa_remove = _std_remove
376
456
 
377
- def _std_copy(self, source, destination,
378
- smartcp_threshold=0, intent='in', pack=False, silent=False):
457
+ def _std_copy(
458
+ self,
459
+ source,
460
+ destination,
461
+ smartcp_threshold=0,
462
+ intent="in",
463
+ pack=False,
464
+ silent=False,
465
+ ):
379
466
  """Extended copy for (possibly) multi lfi file."""
380
467
  st = self._std_prepare(source, destination, intent)
381
468
  if st.rc == 0:
@@ -385,23 +472,48 @@ class LFI_Tool_Raw(addons.FtrawEnableAddon):
385
472
  actual_intent = intent
386
473
  if ln_cross_users and not self.sh.allow_cross_users_links:
387
474
  actual_pack = True
388
- actual_intent = 'inout'
475
+ actual_intent = "inout"
389
476
  try:
390
- st.rc = self._spawn_wrap('copy', (['-pack', ] if actual_pack else []) +
391
- ['-intent={}'.format(actual_intent), source, destination],
392
- output=False)
477
+ st.rc = self._spawn_wrap(
478
+ "copy",
479
+ (
480
+ [
481
+ "-pack",
482
+ ]
483
+ if actual_pack
484
+ else []
485
+ )
486
+ + [
487
+ "-intent={}".format(actual_intent),
488
+ source,
489
+ destination,
490
+ ],
491
+ output=False,
492
+ )
393
493
  except systems.ExecutionError:
394
494
  if self.sh.allow_cross_users_links and ln_cross_users:
395
495
  # This is expected to fail if the fs.protected_hardlinks
396
496
  # Linux kernel setting is 1.
397
- logger.info("Force System's allow_cross_users_links to False")
497
+ logger.info(
498
+ "Force System's allow_cross_users_links to False"
499
+ )
398
500
  self.sh.allow_cross_users_links = False
399
501
  logger.info("Re-running the cp command on this LFI file")
400
- st = self._std_copy(source, destination, intent=intent, pack=pack, silent=silent)
502
+ st = self._std_copy(
503
+ source,
504
+ destination,
505
+ intent=intent,
506
+ pack=pack,
507
+ silent=silent,
508
+ )
401
509
  else:
402
510
  raise
403
511
  else:
404
- if ln_cross_users and not self.sh.allow_cross_users_links and intent == 'in':
512
+ if (
513
+ ln_cross_users
514
+ and not self.sh.allow_cross_users_links
515
+ and intent == "in"
516
+ ):
405
517
  self.sh.readonly(destination)
406
518
  return st
407
519
 
@@ -411,12 +523,23 @@ class LFI_Tool_Raw(addons.FtrawEnableAddon):
411
523
  """Extended mv for (possibly) multi lfi file."""
412
524
  if self.is_xlfi(source):
413
525
  if intent is None:
414
- intent = 'inout' if self.sh.access(source, self.sh.W_OK) else 'in'
526
+ intent = (
527
+ "inout" if self.sh.access(source, self.sh.W_OK) else "in"
528
+ )
415
529
  st = self._std_prepare(source, destination, intent)
416
530
  if st.rc == 0:
417
- st.rc = self._spawn_wrap('move', (['-pack', ] if pack else []) +
418
- ['-intent={}'.format(intent), source, destination],
419
- output=False)
531
+ st.rc = self._spawn_wrap(
532
+ "move",
533
+ (
534
+ [
535
+ "-pack",
536
+ ]
537
+ if pack
538
+ else []
539
+ )
540
+ + ["-intent={}".format(intent), source, destination],
541
+ output=False,
542
+ )
420
543
  else:
421
544
  st = LFI_Status()
422
545
  st.rc = self.sh.mv(source, destination)
@@ -424,27 +547,35 @@ class LFI_Tool_Raw(addons.FtrawEnableAddon):
424
547
 
425
548
  lfi_mv = lfi_move = fa_mv = fa_move = _std_move
426
549
 
427
- def _std_scpput(self, source, destination, hostname, logname=None, cpipeline=None):
550
+ def _std_scpput(
551
+ self, source, destination, hostname, logname=None, cpipeline=None
552
+ ):
428
553
  """On the fly packing and scp."""
429
554
  if not self.is_xlfi(source):
430
- rc = self.sh.scpput(source, destination, hostname, logname, cpipeline)
555
+ rc = self.sh.scpput(
556
+ source, destination, hostname, logname, cpipeline
557
+ )
431
558
  else:
432
559
  if cpipeline is not None:
433
560
  raise OSError("It's not allowed to compress xlfi files.")
434
- logname = self.sh.fix_ftuser(hostname, logname, fatal=False, defaults_to_user=False)
561
+ logname = self.sh.fix_ftuser(
562
+ hostname, logname, fatal=False, defaults_to_user=False
563
+ )
435
564
  ssh = self.sh.ssh(hostname, logname)
436
565
  permissions = ssh.get_permissions(source)
437
566
  # remove the .d companion directory (scp_stream removes the destination)
438
567
  # go on on failure : the .d lingers on, but the lfi will be self-contained
439
- ssh.remove(destination + '.d')
568
+ ssh.remove(destination + ".d")
440
569
  p = self._pack_stream(source)
441
- rc = ssh.scpput_stream(p.stdout, destination, permissions=permissions)
570
+ rc = ssh.scpput_stream(
571
+ p.stdout, destination, permissions=permissions
572
+ )
442
573
  self.sh.pclose(p)
443
574
  return rc
444
575
 
445
576
  fa_scpput = lfi_scpput = _std_scpput
446
577
 
447
- @addons.require_external_addon('ecfs')
578
+ @addons.require_external_addon("ecfs")
448
579
  def _std_ecfsput(self, source, target, cpipeline=None, options=None):
449
580
  """
450
581
  :param source: source file
@@ -459,29 +590,38 @@ class LFI_Tool_Raw(addons.FtrawEnableAddon):
459
590
  psource = self.sh.safe_fileaddsuffix(source)
460
591
  rc = LFI_Status()
461
592
  try:
462
- st = self._std_copy(source=source,
463
- destination=psource,
464
- pack=True)
593
+ st = self._std_copy(
594
+ source=source, destination=psource, pack=True
595
+ )
465
596
  rc = rc and st.rc
466
597
  dict_args = dict()
467
598
  if rc:
468
- rc, dict_args = self.sh.ecfsput(source=psource,
469
- target=target,
470
- options=options)
599
+ rc, dict_args = self.sh.ecfsput(
600
+ source=psource, target=target, options=options
601
+ )
471
602
  finally:
472
603
  self.sh.rm(psource)
473
604
  return rc, dict_args
474
605
  else:
475
- return self.sh.ecfsput(source=source,
476
- target=target,
477
- options=options,
478
- cpipeline=cpipeline)
606
+ return self.sh.ecfsput(
607
+ source=source,
608
+ target=target,
609
+ options=options,
610
+ cpipeline=cpipeline,
611
+ )
479
612
 
480
613
  fa_ecfsput = lfi_ecfsput = _std_ecfsput
481
614
 
482
- @addons.require_external_addon('ectrans')
483
- def _std_ectransput(self, source, target, gateway=None, remote=None,
484
- cpipeline=None, sync=False):
615
+ @addons.require_external_addon("ectrans")
616
+ def _std_ectransput(
617
+ self,
618
+ source,
619
+ target,
620
+ gateway=None,
621
+ remote=None,
622
+ cpipeline=None,
623
+ sync=False,
624
+ ):
485
625
  """
486
626
  :param source: source file
487
627
  :param target: target file
@@ -497,27 +637,31 @@ class LFI_Tool_Raw(addons.FtrawEnableAddon):
497
637
  psource = self.sh.safe_fileaddsuffix(source)
498
638
  rc = LFI_Status()
499
639
  try:
500
- st = self._std_copy(source=source,
501
- destination=psource,
502
- pack=True)
640
+ st = self._std_copy(
641
+ source=source, destination=psource, pack=True
642
+ )
503
643
  rc = rc and st.rc
504
644
  dict_args = dict()
505
645
  if rc:
506
- rc, dict_args = self.sh.raw_ectransput(source=psource,
507
- target=target,
508
- gateway=gateway,
509
- remote=remote,
510
- sync=sync)
646
+ rc, dict_args = self.sh.raw_ectransput(
647
+ source=psource,
648
+ target=target,
649
+ gateway=gateway,
650
+ remote=remote,
651
+ sync=sync,
652
+ )
511
653
  finally:
512
654
  self.sh.rm(psource)
513
655
  return rc, dict_args
514
656
  else:
515
- return self.sh.ectransput(source=source,
516
- target=target,
517
- gateway=gateway,
518
- remote=remote,
519
- cpipeline=cpipeline,
520
- sync=sync)
657
+ return self.sh.ectransput(
658
+ source=source,
659
+ target=target,
660
+ gateway=gateway,
661
+ remote=remote,
662
+ cpipeline=cpipeline,
663
+ sync=sync,
664
+ )
521
665
 
522
666
  fa_ectransput = lfi_ectransput = _std_ectransput
523
667
 
@@ -530,20 +674,23 @@ class LFI_Tool_Py(LFI_Tool_Raw):
530
674
  """
531
675
 
532
676
  _footprint = dict(
533
- info = 'Default LFI system interface (Python)',
534
- attr = dict(
535
- wraplanguage = dict(
536
- values = ['python', ],
677
+ info="Default LFI system interface (Python)",
678
+ attr=dict(
679
+ wraplanguage=dict(
680
+ values=[
681
+ "python",
682
+ ],
537
683
  ),
538
- )
684
+ ),
539
685
  )
540
686
 
541
687
  def _pack_stream(self, source):
542
- return self._spawn(['lfi_alt_pack', '--lfi-file-in', source,
543
- '--lfi-file-out', '-'],
544
- output=False,
545
- inpipe=True,
546
- bufsize=8192)
688
+ return self._spawn(
689
+ ["lfi_alt_pack", "--lfi-file-in", source, "--lfi-file-out", "-"],
690
+ output=False,
691
+ inpipe=True,
692
+ bufsize=8192,
693
+ )
547
694
 
548
695
  def _std_remove(self, *args):
549
696
  """Remove (possibly) multi lfi files."""
@@ -552,12 +699,14 @@ class LFI_Tool_Py(LFI_Tool_Raw):
552
699
  for objpath in self.sh.glob(pname):
553
700
  xlfi = self.is_xlfi(objpath)
554
701
  if xlfi:
555
- rc = self._spawn(['lfi_alt_remv', '--lfi-file', objpath], output=False)
702
+ rc = self._spawn(
703
+ ["lfi_alt_remv", "--lfi-file", objpath], output=False
704
+ )
556
705
  else:
557
706
  rc = self.sh.remove(objpath)
558
707
  st.result.append(dict(path=objpath, multi=xlfi, rc=rc))
559
708
  st.rc = rc
560
- for dirpath in self.sh.glob(pname + '.d'):
709
+ for dirpath in self.sh.glob(pname + ".d"):
561
710
  if self.sh.path.exists(dirpath):
562
711
  rc = self.sh.remove(dirpath)
563
712
  st.result.append(dict(path=dirpath, multi=True, rc=rc))
@@ -568,29 +717,61 @@ class LFI_Tool_Py(LFI_Tool_Raw):
568
717
 
569
718
  def _cp_pack_read(self, source, destination):
570
719
  if self.warnpack:
571
- logger.warning('Suspicious packing <%s>', source)
572
- rc = self._spawn(['lfi_alt_pack', '--lfi-file-in', source, '--lfi-file-out', destination],
573
- output=False)
720
+ logger.warning("Suspicious packing <%s>", source)
721
+ rc = self._spawn(
722
+ [
723
+ "lfi_alt_pack",
724
+ "--lfi-file-in",
725
+ source,
726
+ "--lfi-file-out",
727
+ destination,
728
+ ],
729
+ output=False,
730
+ )
574
731
  self.sh.chmod(destination, 0o444)
575
732
  return rc
576
733
 
577
734
  def _cp_pack_write(self, source, destination):
578
735
  if self.warnpack:
579
- logger.warning('Suspicious packing <%s>', source)
580
- rc = self._spawn(['lfi_alt_pack', '--lfi-file-in', source, '--lfi-file-out', destination],
581
- output=False)
736
+ logger.warning("Suspicious packing <%s>", source)
737
+ rc = self._spawn(
738
+ [
739
+ "lfi_alt_pack",
740
+ "--lfi-file-in",
741
+ source,
742
+ "--lfi-file-out",
743
+ destination,
744
+ ],
745
+ output=False,
746
+ )
582
747
  self.sh.chmod(destination, 0o644)
583
748
  return rc
584
749
 
585
750
  def _cp_copy_read(self, source, destination):
586
- rc = self._spawn(['lfi_alt_copy', '--lfi-file-in', source, '--lfi-file-out', destination],
587
- output=False)
751
+ rc = self._spawn(
752
+ [
753
+ "lfi_alt_copy",
754
+ "--lfi-file-in",
755
+ source,
756
+ "--lfi-file-out",
757
+ destination,
758
+ ],
759
+ output=False,
760
+ )
588
761
  self.sh.chmod(destination, 0o444)
589
762
  return rc
590
763
 
591
764
  def _cp_copy_write(self, source, destination):
592
- rc = self._spawn(['lfi_alt_copy', '--lfi-file-in', source, '--lfi-file-out', destination],
593
- output=False)
765
+ rc = self._spawn(
766
+ [
767
+ "lfi_alt_copy",
768
+ "--lfi-file-in",
769
+ source,
770
+ "--lfi-file-out",
771
+ destination,
772
+ ],
773
+ output=False,
774
+ )
594
775
  self.sh.chmod(destination, 0o644)
595
776
  return rc
596
777
 
@@ -604,38 +785,55 @@ class LFI_Tool_Py(LFI_Tool_Raw):
604
785
  _cp_nopack_fsko_read = _cp_pack_read
605
786
  _cp_nopack_fsko_write = _cp_pack_write
606
787
 
607
- def _multicpmethod(self, pack=False, intent='in', samefs=False):
608
- return '_cp_{:s}_{:s}_{:s}'.format(
609
- 'aspack' if pack else 'nopack',
610
- 'fsok' if samefs else 'fsko',
611
- 'read' if intent == 'in' else 'write',
788
+ def _multicpmethod(self, pack=False, intent="in", samefs=False):
789
+ return "_cp_{:s}_{:s}_{:s}".format(
790
+ "aspack" if pack else "nopack",
791
+ "fsok" if samefs else "fsko",
792
+ "read" if intent == "in" else "write",
612
793
  )
613
794
 
614
- def _std_copy(self, source, destination,
615
- smartcp_threshold=0, intent='in', pack=False, silent=False):
795
+ def _std_copy(
796
+ self,
797
+ source,
798
+ destination,
799
+ smartcp_threshold=0,
800
+ intent="in",
801
+ pack=False,
802
+ silent=False,
803
+ ):
616
804
  """Extended copy for (possibly) multi lfi file."""
617
805
  st = LFI_Status()
618
806
  if not self.sh.path.exists(source):
619
- logger.error('Missing source %s', source)
807
+ logger.error("Missing source %s", source)
620
808
  st.rc = 2
621
- st.stderr = 'No such source file or directory : [' + source + ']'
809
+ st.stderr = "No such source file or directory : [" + source + "]"
622
810
  return st
623
811
  if self.is_xlfi(source):
624
812
  if not self.sh.filecocoon(destination):
625
- raise OSError('Could not cocoon [' + destination + ']')
813
+ raise OSError("Could not cocoon [" + destination + "]")
626
814
  if not self.lfi_rm(destination):
627
- raise OSError('Could not clean destination [' + destination + ']')
628
- xcp = self._multicpmethod(pack=pack, intent=intent, samefs=self.sh.is_samefs(source, destination))
815
+ raise OSError(
816
+ "Could not clean destination [" + destination + "]"
817
+ )
818
+ xcp = self._multicpmethod(
819
+ pack=pack,
820
+ intent=intent,
821
+ samefs=self.sh.is_samefs(source, destination),
822
+ )
629
823
  actualcp = getattr(self, xcp, None)
630
824
  if actualcp is None:
631
- raise AttributeError('No actual LFI cp command ' + xcp)
825
+ raise AttributeError("No actual LFI cp command " + xcp)
632
826
  else:
633
827
  st.rc = actualcp(source, self.sh.path.realpath(destination))
634
828
  else:
635
- if intent == 'in':
636
- st.rc = self.sh.smartcp(source, destination, smartcp_threshold=smartcp_threshold)
829
+ if intent == "in":
830
+ st.rc = self.sh.smartcp(
831
+ source, destination, smartcp_threshold=smartcp_threshold
832
+ )
637
833
  else:
638
- st.rc = self.sh.cp(source, destination, smartcp_threshold=smartcp_threshold)
834
+ st.rc = self.sh.cp(
835
+ source, destination, smartcp_threshold=smartcp_threshold
836
+ )
639
837
  return st
640
838
 
641
839
  lfi_cp = lfi_copy = fa_cp = fa_copy = _std_copy
@@ -644,7 +842,9 @@ class LFI_Tool_Py(LFI_Tool_Raw):
644
842
  """Extended mv for (possibly) multi lfi file."""
645
843
  if self.is_xlfi(source):
646
844
  if intent is None:
647
- intent = 'inout' if self.sh.access(source, self.sh.W_OK) else 'in'
845
+ intent = (
846
+ "inout" if self.sh.access(source, self.sh.W_OK) else "in"
847
+ )
648
848
  st = self.lfi_cp(source, destination, intent=intent, pack=pack)
649
849
  if st:
650
850
  st = self.lfi_rm(source)
@@ -662,72 +862,79 @@ class IO_Poll(addons.Addon):
662
862
  This addon is in charge of multi-file reshaping after IFS-ARPEGE execution.
663
863
  """
664
864
 
665
- LFI_HNDL_SPEC = ':1'
865
+ LFI_HNDL_SPEC = ":1"
666
866
  DR_HOOK_SILENT = 1
667
867
  DR_HOOK_NOT_MPI = 1
668
868
  DR_HOOK_ASSERT_MPI_INITIALIZED = 0
669
- OMP_STACKSIZE = '32M'
670
- KMP_STACKSIZE = '32M'
671
- KMP_MONITOR_STACKSIZE = '32M'
869
+ OMP_STACKSIZE = "32M"
870
+ KMP_STACKSIZE = "32M"
871
+ KMP_MONITOR_STACKSIZE = "32M"
672
872
 
673
873
  _footprint = dict(
674
- info = 'Default io_poll system interface',
675
- attr = dict(
676
- kind = dict(
677
- values = ['iopoll', 'io_poll'],
678
- remap = dict(
679
- io_poll = 'iopoll',
680
- )
874
+ info="Default io_poll system interface",
875
+ attr=dict(
876
+ kind=dict(
877
+ values=["iopoll", "io_poll"],
878
+ remap=dict(
879
+ io_poll="iopoll",
880
+ ),
681
881
  ),
682
- cfginfo = dict(
683
- default = 'lfi',
882
+ cfginfo=dict(
883
+ default="lfi",
684
884
  ),
685
- cmd = dict(
686
- alias = ('iopollcmd', 'io_pollcmd', 'io_poll_cmd'),
687
- default = 'io_poll',
885
+ cmd=dict(
886
+ alias=("iopollcmd", "io_pollcmd", "io_poll_cmd"),
887
+ default="io_poll",
688
888
  ),
689
- path = dict(
690
- alias = ('iopollpath', 'io_pollpath', 'io_poll_path', 'iopath'),
889
+ path=dict(
890
+ alias=("iopollpath", "io_pollpath", "io_poll_path", "iopath"),
691
891
  ),
692
- interpreter = dict(
693
- values = ['perl', 'none', 'None'],
694
- remap = dict({'None': 'none', }),
695
- default = 'perl',
696
- optional = True,
892
+ interpreter=dict(
893
+ values=["perl", "none", "None"],
894
+ remap=dict(
895
+ {
896
+ "None": "none",
897
+ }
898
+ ),
899
+ default="perl",
900
+ optional=True,
697
901
  ),
698
- toolkind = dict(
699
- default = 'iopoll'
700
- )
701
- )
902
+ toolkind=dict(default="iopoll"),
903
+ ),
702
904
  )
703
905
 
704
906
  def __init__(self, *args, **kw):
705
907
  """Abstract Addon initialisation."""
706
- logger.debug('IO_Poll init %s', self.__class__)
908
+ logger.debug("IO_Poll init %s", self.__class__)
707
909
  super().__init__(*args, **kw)
708
910
  self._polled = set()
709
911
 
710
912
  def _spawn(self, cmd, **kw):
711
913
  """Tube to set LFITOOLS env variable."""
712
- if 'LFITOOLS' not in self.env:
713
- active_lfi = (LFI_Tool_Raw.in_shell(self.sh) or
714
- LFI_Tool_Py.in_shell(self.sh))
914
+ if "LFITOOLS" not in self.env:
915
+ active_lfi = LFI_Tool_Raw.in_shell(
916
+ self.sh
917
+ ) or LFI_Tool_Py.in_shell(self.sh)
715
918
  if active_lfi is None:
716
- raise RuntimeError('Could not find any active LFI Tool')
717
- self.env.LFITOOLS = active_lfi.actual_path + '/' + active_lfi.actual_cmd
919
+ raise RuntimeError("Could not find any active LFI Tool")
920
+ self.env.LFITOOLS = (
921
+ active_lfi.actual_path + "/" + active_lfi.actual_cmd
922
+ )
718
923
  # Is there a need for an interpreter ?
719
- if self.interpreter != 'none':
720
- kw['interpreter'] = self.interpreter
924
+ if self.interpreter != "none":
925
+ kw["interpreter"] = self.interpreter
721
926
  return super()._spawn(cmd, **kw)
722
927
 
723
928
  def io_poll(self, prefix, nproc_io=None):
724
929
  """Do the actual job of polling files prefixed by ``prefix``."""
725
- cmd = ['--prefix', prefix]
930
+ cmd = ["--prefix", prefix]
726
931
  if nproc_io is None:
727
- if not self.sh.path.exists('fort.4'):
728
- raise OSError('The proc_io option or a fort.4 file should be provided.')
932
+ if not self.sh.path.exists("fort.4"):
933
+ raise OSError(
934
+ "The proc_io option or a fort.4 file should be provided."
935
+ )
729
936
  else:
730
- cmd.extend(['--nproc_io', str(nproc_io)])
937
+ cmd.extend(["--nproc_io", str(nproc_io)])
731
938
 
732
939
  # Catch the file processed
733
940
  rawout = self._spawn(cmd)