vortex-nwp 2.0.0b1__py3-none-any.whl → 2.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (141) hide show
  1. vortex/__init__.py +75 -47
  2. vortex/algo/__init__.py +3 -2
  3. vortex/algo/components.py +944 -618
  4. vortex/algo/mpitools.py +802 -497
  5. vortex/algo/mpitools_templates/__init__.py +1 -0
  6. vortex/algo/serversynctools.py +34 -33
  7. vortex/config.py +19 -22
  8. vortex/data/__init__.py +9 -3
  9. vortex/data/abstractstores.py +593 -655
  10. vortex/data/containers.py +217 -162
  11. vortex/data/contents.py +65 -39
  12. vortex/data/executables.py +93 -102
  13. vortex/data/flow.py +40 -34
  14. vortex/data/geometries.py +228 -132
  15. vortex/data/handlers.py +436 -227
  16. vortex/data/outflow.py +15 -15
  17. vortex/data/providers.py +185 -163
  18. vortex/data/resources.py +48 -42
  19. vortex/data/stores.py +540 -417
  20. vortex/data/sync_templates/__init__.py +0 -0
  21. vortex/gloves.py +114 -87
  22. vortex/layout/__init__.py +1 -8
  23. vortex/layout/contexts.py +150 -84
  24. vortex/layout/dataflow.py +353 -202
  25. vortex/layout/monitor.py +264 -128
  26. vortex/nwp/__init__.py +5 -2
  27. vortex/nwp/algo/__init__.py +14 -5
  28. vortex/nwp/algo/assim.py +205 -151
  29. vortex/nwp/algo/clim.py +683 -517
  30. vortex/nwp/algo/coupling.py +447 -225
  31. vortex/nwp/algo/eda.py +437 -229
  32. vortex/nwp/algo/eps.py +403 -231
  33. vortex/nwp/algo/forecasts.py +416 -275
  34. vortex/nwp/algo/fpserver.py +683 -307
  35. vortex/nwp/algo/ifsnaming.py +205 -145
  36. vortex/nwp/algo/ifsroot.py +215 -122
  37. vortex/nwp/algo/monitoring.py +137 -76
  38. vortex/nwp/algo/mpitools.py +330 -190
  39. vortex/nwp/algo/odbtools.py +637 -353
  40. vortex/nwp/algo/oopsroot.py +454 -273
  41. vortex/nwp/algo/oopstests.py +90 -56
  42. vortex/nwp/algo/request.py +287 -206
  43. vortex/nwp/algo/stdpost.py +878 -522
  44. vortex/nwp/data/__init__.py +22 -4
  45. vortex/nwp/data/assim.py +125 -137
  46. vortex/nwp/data/boundaries.py +121 -68
  47. vortex/nwp/data/climfiles.py +193 -211
  48. vortex/nwp/data/configfiles.py +73 -69
  49. vortex/nwp/data/consts.py +426 -401
  50. vortex/nwp/data/ctpini.py +59 -43
  51. vortex/nwp/data/diagnostics.py +94 -66
  52. vortex/nwp/data/eda.py +50 -51
  53. vortex/nwp/data/eps.py +195 -146
  54. vortex/nwp/data/executables.py +440 -434
  55. vortex/nwp/data/fields.py +63 -48
  56. vortex/nwp/data/gridfiles.py +183 -111
  57. vortex/nwp/data/logs.py +250 -217
  58. vortex/nwp/data/modelstates.py +180 -151
  59. vortex/nwp/data/monitoring.py +72 -99
  60. vortex/nwp/data/namelists.py +254 -202
  61. vortex/nwp/data/obs.py +400 -308
  62. vortex/nwp/data/oopsexec.py +22 -20
  63. vortex/nwp/data/providers.py +90 -65
  64. vortex/nwp/data/query.py +71 -82
  65. vortex/nwp/data/stores.py +49 -36
  66. vortex/nwp/data/surfex.py +136 -137
  67. vortex/nwp/syntax/__init__.py +1 -1
  68. vortex/nwp/syntax/stdattrs.py +173 -111
  69. vortex/nwp/tools/__init__.py +2 -2
  70. vortex/nwp/tools/addons.py +22 -17
  71. vortex/nwp/tools/agt.py +24 -12
  72. vortex/nwp/tools/bdap.py +16 -5
  73. vortex/nwp/tools/bdcp.py +4 -1
  74. vortex/nwp/tools/bdm.py +3 -0
  75. vortex/nwp/tools/bdmp.py +14 -9
  76. vortex/nwp/tools/conftools.py +728 -378
  77. vortex/nwp/tools/drhook.py +12 -8
  78. vortex/nwp/tools/grib.py +65 -39
  79. vortex/nwp/tools/gribdiff.py +22 -17
  80. vortex/nwp/tools/ifstools.py +82 -42
  81. vortex/nwp/tools/igastuff.py +167 -143
  82. vortex/nwp/tools/mars.py +14 -2
  83. vortex/nwp/tools/odb.py +234 -125
  84. vortex/nwp/tools/partitioning.py +61 -37
  85. vortex/nwp/tools/satrad.py +27 -12
  86. vortex/nwp/util/async.py +83 -55
  87. vortex/nwp/util/beacon.py +10 -10
  88. vortex/nwp/util/diffpygram.py +174 -86
  89. vortex/nwp/util/ens.py +144 -63
  90. vortex/nwp/util/hooks.py +30 -19
  91. vortex/nwp/util/taskdeco.py +28 -24
  92. vortex/nwp/util/usepygram.py +278 -172
  93. vortex/nwp/util/usetnt.py +31 -17
  94. vortex/sessions.py +72 -39
  95. vortex/syntax/__init__.py +1 -1
  96. vortex/syntax/stdattrs.py +410 -171
  97. vortex/syntax/stddeco.py +31 -22
  98. vortex/toolbox.py +327 -192
  99. vortex/tools/__init__.py +11 -2
  100. vortex/tools/actions.py +110 -121
  101. vortex/tools/addons.py +111 -92
  102. vortex/tools/arm.py +42 -22
  103. vortex/tools/compression.py +72 -69
  104. vortex/tools/date.py +11 -4
  105. vortex/tools/delayedactions.py +242 -132
  106. vortex/tools/env.py +75 -47
  107. vortex/tools/folder.py +342 -171
  108. vortex/tools/grib.py +341 -162
  109. vortex/tools/lfi.py +423 -216
  110. vortex/tools/listings.py +109 -40
  111. vortex/tools/names.py +218 -156
  112. vortex/tools/net.py +655 -299
  113. vortex/tools/parallelism.py +93 -61
  114. vortex/tools/prestaging.py +55 -31
  115. vortex/tools/schedulers.py +172 -105
  116. vortex/tools/services.py +403 -334
  117. vortex/tools/storage.py +293 -358
  118. vortex/tools/surfex.py +24 -24
  119. vortex/tools/systems.py +1234 -643
  120. vortex/tools/targets.py +156 -100
  121. vortex/util/__init__.py +1 -1
  122. vortex/util/config.py +378 -327
  123. vortex/util/empty.py +2 -2
  124. vortex/util/helpers.py +56 -24
  125. vortex/util/introspection.py +18 -12
  126. vortex/util/iosponge.py +8 -4
  127. vortex/util/roles.py +4 -6
  128. vortex/util/storefunctions.py +39 -13
  129. vortex/util/structs.py +3 -3
  130. vortex/util/worker.py +29 -17
  131. vortex_nwp-2.1.0.dist-info/METADATA +67 -0
  132. vortex_nwp-2.1.0.dist-info/RECORD +144 -0
  133. {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.1.0.dist-info}/WHEEL +1 -1
  134. vortex/layout/appconf.py +0 -109
  135. vortex/layout/jobs.py +0 -1276
  136. vortex/layout/nodes.py +0 -1424
  137. vortex/layout/subjobs.py +0 -464
  138. vortex_nwp-2.0.0b1.dist-info/METADATA +0 -50
  139. vortex_nwp-2.0.0b1.dist-info/RECORD +0 -146
  140. {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.1.0.dist-info/licenses}/LICENSE +0 -0
  141. {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.1.0.dist-info}/top_level.txt +0 -0
vortex/tools/systems.py CHANGED
@@ -67,6 +67,8 @@ from vortex.tools.compression import CompressionPipeline
67
67
  from vortex.tools.env import Environment
68
68
  from vortex.tools.net import AssistedSsh, AutoRetriesFtp, DEFAULT_FTP_PORT
69
69
  from vortex.tools.net import FtpConnectionPool, LinuxNetstats, StdFtp
70
+ import vortex.tools.storage
71
+ from vortex import config
70
72
 
71
73
  #: No automatic export
72
74
  __all__ = []
@@ -74,13 +76,13 @@ __all__ = []
74
76
  logger = loggers.getLogger(__name__)
75
77
 
76
78
  #: Pre-compiled regex to check a none str value
77
- isnonedef = re.compile(r'\s*none\s*$', re.IGNORECASE)
79
+ isnonedef = re.compile(r"\s*none\s*$", re.IGNORECASE)
78
80
 
79
81
  #: Pre-compiled regex to check a boolean true str value
80
- istruedef = re.compile(r'\s*(on|true|ok)\s*$', re.IGNORECASE)
82
+ istruedef = re.compile(r"\s*(on|true|ok)\s*$", re.IGNORECASE)
81
83
 
82
84
  #: Pre-compiled regex to check a boolean false str value
83
- isfalsedef = re.compile(r'\s*(off|false|ko)\s*$', re.IGNORECASE)
85
+ isfalsedef = re.compile(r"\s*(off|false|ko)\s*$", re.IGNORECASE)
84
86
 
85
87
  #: Global lock to protect temporary locale changes
86
88
  LOCALE_LOCK = threading.Lock()
@@ -97,7 +99,9 @@ _fmtshcmd_docbonus = """
97
99
  # Constant items
98
100
 
99
101
  #: Definition of a named tuple ftpflavour
100
- FtpFlavourTuple = namedtuple('FtpFlavourTuple', ['STD', 'RETRIES', 'CONNECTION_POOLS'])
102
+ FtpFlavourTuple = namedtuple(
103
+ "FtpFlavourTuple", ["STD", "RETRIES", "CONNECTION_POOLS"]
104
+ )
101
105
 
102
106
  #: Predefined FTP_FLAVOUR values IN, OUT and INOUT.
103
107
  FTP_FLAVOUR = FtpFlavourTuple(STD=0, RETRIES=1, CONNECTION_POOLS=2)
@@ -114,10 +118,12 @@ def fmtshcmd(func):
114
118
  """
115
119
 
116
120
  def formatted_method(self, *args, **kw):
117
- fmt = kw.pop('fmt', None)
121
+ fmt = kw.pop("fmt", None)
118
122
  shtarget = self if isinstance(self, System) else self.sh
119
- fmtcall = getattr(shtarget, str(fmt).lower() + '_' + func.__name__, func)
120
- if getattr(fmtcall, 'func_extern', False):
123
+ fmtcall = getattr(
124
+ shtarget, str(fmt).lower() + "_" + func.__name__, func
125
+ )
126
+ if getattr(fmtcall, "func_extern", False):
121
127
  return fmtcall(*args, **kw)
122
128
  else:
123
129
  return fmtcall(self, *args, **kw)
@@ -144,11 +150,13 @@ def _kw2spawn(func):
144
150
 
145
151
  class ExecutionError(RuntimeError):
146
152
  """Go through exception for internal :meth:`OSExtended.spawn` errors."""
153
+
147
154
  pass
148
155
 
149
156
 
150
157
  class CopyTreeError(OSError):
151
158
  """An error raised during the recursive copy of a directory."""
159
+
152
160
  pass
153
161
 
154
162
 
@@ -171,12 +179,12 @@ class CdContext:
171
179
  self.newpath = self.sh.path.expanduser(newpath)
172
180
 
173
181
  def __enter__(self):
174
- if self.newpath not in ('', '.'):
182
+ if self.newpath not in ("", "."):
175
183
  self.oldpath = self.sh.getcwd()
176
184
  self.sh.cd(self.newpath, create=self.create)
177
185
 
178
186
  def __exit__(self, etype, value, traceback): # @UnusedVariable
179
- if self.newpath not in ('', '.'):
187
+ if self.newpath not in ("", "."):
180
188
  self.sh.cd(self.oldpath)
181
189
  if self.clean_onexit:
182
190
  self.sh.rm(self.newpath)
@@ -220,14 +228,14 @@ class PythonSimplifiedVersion:
220
228
  It can be used in a footprint specification.
221
229
  """
222
230
 
223
- _VERSION_RE = re.compile(r'(\d+)\.(\d+)\.(\d+)')
231
+ _VERSION_RE = re.compile(r"(\d+)\.(\d+)\.(\d+)")
224
232
 
225
233
  def __init__(self, versionstr):
226
234
  v_match = self._VERSION_RE.match(versionstr)
227
235
  if v_match:
228
236
  self._version = tuple([int(d) for d in v_match.groups()])
229
237
  else:
230
- raise ValueError('Malformed version string: {}'.format(versionstr))
238
+ raise ValueError("Malformed version string: {}".format(versionstr))
231
239
 
232
240
  @property
233
241
  def version(self):
@@ -250,10 +258,12 @@ class PythonSimplifiedVersion:
250
258
  return self.version > other.version
251
259
 
252
260
  def __str__(self):
253
- return '.'.join([str(d) for d in self.version])
261
+ return ".".join([str(d) for d in self.version])
254
262
 
255
263
  def __repr__(self):
256
- return '<{} | {!s}>'.format(object.__repr__(self).lstrip('<').rstrip('>'), self)
264
+ return "<{} | {!s}>".format(
265
+ object.__repr__(self).lstrip("<").rstrip(">"), self
266
+ )
257
267
 
258
268
  def export_dict(self):
259
269
  """The pure dict/json output is the raw integer"""
@@ -269,50 +279,50 @@ class System(footprints.FootprintBase):
269
279
 
270
280
  _abstract = True
271
281
  _explicit = False
272
- _collector = ('system',)
282
+ _collector = ("system",)
273
283
 
274
284
  _footprint = dict(
275
- info = 'Default system interface',
276
- attr = dict(
277
- hostname = dict(
278
- info = "The computer's network name",
279
- optional = True,
280
- default = platform.node(),
281
- alias = ('nodename',)
285
+ info="Default system interface",
286
+ attr=dict(
287
+ hostname=dict(
288
+ info="The computer's network name",
289
+ optional=True,
290
+ default=platform.node(),
291
+ alias=("nodename",),
282
292
  ),
283
- sysname = dict(
284
- info = "The underlying system/OS name (e.g. Linux, Darwin, ...)",
285
- optional = True,
286
- default = platform.system(),
293
+ sysname=dict(
294
+ info="The underlying system/OS name (e.g. Linux, Darwin, ...)",
295
+ optional=True,
296
+ default=platform.system(),
287
297
  ),
288
- arch = dict(
289
- info = "The underlying machine type (e.g. i386, x86_64, ...)",
290
- optional = True,
291
- default = platform.machine(),
292
- alias = ('machine',)
298
+ arch=dict(
299
+ info="The underlying machine type (e.g. i386, x86_64, ...)",
300
+ optional=True,
301
+ default=platform.machine(),
302
+ alias=("machine",),
293
303
  ),
294
- release = dict(
295
- info = "The underlying system's release, (e.g. 2.2.0, NT, ...)",
296
- optional = True,
297
- default = platform.release()
304
+ release=dict(
305
+ info="The underlying system's release, (e.g. 2.2.0, NT, ...)",
306
+ optional=True,
307
+ default=platform.release(),
298
308
  ),
299
- version = dict(
300
- info = "The underlying system's release version",
301
- optional = True,
302
- default = platform.version()
309
+ version=dict(
310
+ info="The underlying system's release version",
311
+ optional=True,
312
+ default=platform.version(),
303
313
  ),
304
- python = dict(
305
- info = "The Python's version (e.g 2.7.5)",
306
- type = PythonSimplifiedVersion,
307
- optional = True,
308
- default = platform.python_version(),
314
+ python=dict(
315
+ info="The Python's version (e.g 2.7.5)",
316
+ type=PythonSimplifiedVersion,
317
+ optional=True,
318
+ default=platform.python_version(),
309
319
  ),
310
- glove = dict(
311
- info = "The session's Glove object",
312
- optional = True,
313
- type = Glove,
314
- )
315
- )
320
+ glove=dict(
321
+ info="The session's Glove object",
322
+ optional=True,
323
+ type=Glove,
324
+ ),
325
+ ),
316
326
  )
317
327
 
318
328
  def __init__(self, *args, **kw):
@@ -364,25 +374,29 @@ class System(footprints.FootprintBase):
364
374
  user will be able to call ``sh.greatstuff``).
365
375
 
366
376
  """
367
- logger.debug('Abstract System init %s', self.__class__)
368
- self.__dict__['_os'] = kw.pop('os', os)
369
- self.__dict__['_rl'] = kw.pop('rlimit', resource)
370
- self.__dict__['_sh'] = kw.pop('shutil', kw.pop('sh', shutil))
371
- self.__dict__['_search'] = [self.__dict__['_os'], self.__dict__['_sh'], self.__dict__['_rl']]
372
- self.__dict__['_xtrack'] = dict()
373
- self.__dict__['_history'] = History(tag='shell')
374
- self.__dict__['_rclast'] = 0
375
- self.__dict__['prompt'] = str(kw.pop('prompt', ''))
376
- for flag in ('trace', 'timer'):
377
+ logger.debug("Abstract System init %s", self.__class__)
378
+ self.__dict__["_os"] = kw.pop("os", os)
379
+ self.__dict__["_rl"] = kw.pop("rlimit", resource)
380
+ self.__dict__["_sh"] = kw.pop("shutil", kw.pop("sh", shutil))
381
+ self.__dict__["_search"] = [
382
+ self.__dict__["_os"],
383
+ self.__dict__["_sh"],
384
+ self.__dict__["_rl"],
385
+ ]
386
+ self.__dict__["_xtrack"] = dict()
387
+ self.__dict__["_history"] = History(tag="shell")
388
+ self.__dict__["_rclast"] = 0
389
+ self.__dict__["prompt"] = str(kw.pop("prompt", ""))
390
+ for flag in ("trace", "timer"):
377
391
  self.__dict__[flag] = kw.pop(flag, False)
378
- for flag in ('output',):
392
+ for flag in ("output",):
379
393
  self.__dict__[flag] = kw.pop(flag, True)
380
394
  super().__init__(*args, **kw)
381
395
 
382
396
  @property
383
397
  def realkind(self):
384
398
  """The object/class realkind."""
385
- return 'system'
399
+ return "system"
386
400
 
387
401
  @property
388
402
  def history(self):
@@ -407,18 +421,18 @@ class System(footprints.FootprintBase):
407
421
  @property
408
422
  def default_syslog(self):
409
423
  """Address to use in logging.handler.SysLogHandler()."""
410
- return '/dev/log'
424
+ return "/dev/log"
411
425
 
412
426
  def extend(self, obj=None):
413
427
  """Extend the current external attribute resolution to **obj** (module or object)."""
414
428
  if obj is not None:
415
- if hasattr(obj, 'kind'):
429
+ if hasattr(obj, "kind"):
416
430
  for k, v in self._xtrack.items():
417
- if hasattr(v, 'kind'):
431
+ if hasattr(v, "kind"):
418
432
  if hasattr(self, k):
419
433
  delattr(self, k)
420
434
  for addon in self.search:
421
- if hasattr(addon, 'kind') and addon.kind == obj.kind:
435
+ if hasattr(addon, "kind") and addon.kind == obj.kind:
422
436
  self.search.remove(addon)
423
437
  self.search.append(obj)
424
438
  return len(self.search)
@@ -429,7 +443,7 @@ class System(footprints.FootprintBase):
429
443
  (*i.e.* :class:`~vortex.tools.addons.Addon objects previously
430
444
  loaded with the :meth:`extend` method).
431
445
  """
432
- return [addon.kind for addon in self.search if hasattr(addon, 'kind')]
446
+ return [addon.kind for addon in self.search if hasattr(addon, "kind")]
433
447
 
434
448
  def external(self, key):
435
449
  """Return effective module object reference if any, or *None*."""
@@ -446,12 +460,14 @@ class System(footprints.FootprintBase):
446
460
  This is the place where the ``self.search`` list is looked for...
447
461
  """
448
462
  actualattr = None
449
- if key.startswith('_'):
463
+ if key.startswith("_"):
450
464
  # Do not attempt to look for hidden attributes
451
- raise AttributeError('Method or attribute ' + key + ' not found')
465
+ raise AttributeError("Method or attribute " + key + " not found")
452
466
  for shxobj in self.search:
453
467
  if hasattr(shxobj, key):
454
- if isinstance(shxobj, footprints.FootprintBase) and shxobj.footprint_has_attribute(key):
468
+ if isinstance(
469
+ shxobj, footprints.FootprintBase
470
+ ) and shxobj.footprint_has_attribute(key):
455
471
  # Ignore footprint attributes
456
472
  continue
457
473
  if actualattr is None:
@@ -459,17 +475,24 @@ class System(footprints.FootprintBase):
459
475
  self._xtrack[key] = shxobj
460
476
  else:
461
477
  # Do not warn for a restricted list of keys
462
- if key not in ('stat',):
463
- logger.warning('System: duplicate entry while looking for key="%s". ' +
464
- 'First result in %s but also available in %s.',
465
- key, self._xtrack[key], shxobj)
478
+ if key not in ("stat",):
479
+ logger.warning(
480
+ 'System: duplicate entry while looking for key="%s". '
481
+ + "First result in %s but also available in %s.",
482
+ key,
483
+ self._xtrack[key],
484
+ shxobj,
485
+ )
466
486
  if actualattr is None:
467
- raise AttributeError('Method or attribute ' + key + ' not found')
487
+ raise AttributeError("Method or attribute " + key + " not found")
468
488
  if callable(actualattr):
489
+
469
490
  def osproxy(*args, **kw):
470
491
  cmd = [key]
471
492
  cmd.extend(args)
472
- cmd.extend(['{:s}={:s}'.format(x, str(kw[x])) for x in kw.keys()])
493
+ cmd.extend(
494
+ ["{:s}={:s}".format(x, str(kw[x])) for x in kw.keys()]
495
+ )
473
496
  self.stderr(*cmd)
474
497
  return actualattr(*args, **kw)
475
498
 
@@ -484,15 +507,21 @@ class System(footprints.FootprintBase):
484
507
 
485
508
  def stderr(self, *args):
486
509
  """Write a formatted message to standard error (if ``self.trace == True``)."""
487
- count, justnow, = self.history.append(*args)
510
+ (
511
+ count,
512
+ justnow,
513
+ ) = self.history.append(*args)
488
514
  if self.trace:
489
- if self.trace == 'log':
490
- logger.info('[sh:#%d] %s', count, ' '.join([str(x) for x in args]))
515
+ if self.trace == "log":
516
+ logger.info(
517
+ "[sh:#%d] %s", count, " ".join([str(x) for x in args])
518
+ )
491
519
  else:
492
520
  sys.stderr.write(
493
521
  "* [{:s}][{:d}] {:s}\n".format(
494
- justnow.strftime('%Y/%m/%d-%H:%M:%S'), count,
495
- ' '.join([str(x) for x in args])
522
+ justnow.strftime("%Y/%m/%d-%H:%M:%S"),
523
+ count,
524
+ " ".join([str(x) for x in args]),
496
525
  )
497
526
  )
498
527
 
@@ -512,9 +541,9 @@ class System(footprints.FootprintBase):
512
541
 
513
542
  def echo(self, *args):
514
543
  """Joined **args** are echoed."""
515
- print('>>>', ' '.join([str(arg) for arg in args]))
544
+ print(">>>", " ".join([str(arg) for arg in args]))
516
545
 
517
- def title(self, textlist, tchar='=', autolen=96):
546
+ def title(self, textlist, tchar="=", autolen=96):
518
547
  """Formated title output.
519
548
 
520
549
  :param list|str textlist: A list of strings that contains the title's text
@@ -530,12 +559,16 @@ class System(footprints.FootprintBase):
530
559
  print()
531
560
  print(tchar * (nbc + 4))
532
561
  for text in textlist:
533
- print('{0:s} {1:^{size}s} {0:s}'.format(tchar, text.upper(), size=nbc))
562
+ print(
563
+ "{0:s} {1:^{size}s} {0:s}".format(
564
+ tchar, text.upper(), size=nbc
565
+ )
566
+ )
534
567
  print(tchar * (nbc + 4))
535
568
  print()
536
569
  self.flush_stdall()
537
570
 
538
- def subtitle(self, text='', tchar='-', autolen=96):
571
+ def subtitle(self, text="", tchar="-", autolen=96):
539
572
  """Formatted subtitle output.
540
573
 
541
574
  :param str text: The subtitle's text
@@ -549,11 +582,13 @@ class System(footprints.FootprintBase):
549
582
  print()
550
583
  print(tchar * (nbc + 4))
551
584
  if text:
552
- print('# {0:{size}s} #'.format(text, size=nbc))
585
+ print("# {0:{size}s} #".format(text, size=nbc))
553
586
  print(tchar * (nbc + 4))
554
587
  self.flush_stdall()
555
588
 
556
- def header(self, text='', tchar='-', autolen=False, xline=True, prompt=None):
589
+ def header(
590
+ self, text="", tchar="-", autolen=False, xline=True, prompt=None
591
+ ):
557
592
  """Formatted header output.
558
593
 
559
594
  :param str text: The subtitle's text
@@ -572,15 +607,17 @@ class System(footprints.FootprintBase):
572
607
  if not prompt:
573
608
  prompt = self.prompt
574
609
  if prompt:
575
- prompt = str(prompt) + ' '
610
+ prompt = str(prompt) + " "
576
611
  else:
577
- prompt = ''
612
+ prompt = ""
578
613
  print(prompt + str(text))
579
614
  if xline:
580
615
  print(tchar * nbc)
581
616
  self.flush_stdall()
582
617
 
583
- def highlight(self, text='', hchar='----', bchar='#', bline=False, bline0=True):
618
+ def highlight(
619
+ self, text="", hchar="----", bchar="#", bline=False, bline0=True
620
+ ):
584
621
  """Highlight some text.
585
622
 
586
623
  :param str text: The text to be highlighted
@@ -591,8 +628,11 @@ class System(footprints.FootprintBase):
591
628
  """
592
629
  if bline0:
593
630
  print()
594
- print('{0:s} {1:s} {2:s} {1:s} {3:s}'
595
- .format(bchar.rstrip(), hchar, text, bchar.lstrip()))
631
+ print(
632
+ "{0:s} {1:s} {2:s} {1:s} {3:s}".format(
633
+ bchar.rstrip(), hchar, text, bchar.lstrip()
634
+ )
635
+ )
596
636
  if bline:
597
637
  print()
598
638
  self.flush_stdall()
@@ -600,18 +640,18 @@ class System(footprints.FootprintBase):
600
640
  @property
601
641
  def executable(self):
602
642
  """Return the actual ``sys.executable``."""
603
- self.stderr('executable')
643
+ self.stderr("executable")
604
644
  return sys.executable
605
645
 
606
646
  def pythonpath(self, output=None):
607
647
  """Return or print actual ``sys.path``."""
608
648
  if output is None:
609
649
  output = self.output
610
- self.stderr('pythonpath')
650
+ self.stderr("pythonpath")
611
651
  if output:
612
652
  return sys.path[:]
613
653
  else:
614
- self.subtitle('Python PATH')
654
+ self.subtitle("Python PATH")
615
655
  for pypath in sys.path:
616
656
  print(pypath)
617
657
  return True
@@ -625,12 +665,12 @@ class System(footprints.FootprintBase):
625
665
  """Try to determine an identification string for the current script."""
626
666
  # PBS scheduler SLURM scheduler Good-old PID
627
667
  env = self.env
628
- label = env.PBS_JOBID or env.SLURM_JOB_ID or 'localpid'
629
- if label == 'localpid':
668
+ label = env.PBS_JOBID or env.SLURM_JOB_ID or "localpid"
669
+ if label == "localpid":
630
670
  label = str(self.getpid())
631
671
  return label
632
672
 
633
- def vortex_modules(self, only='.'):
673
+ def vortex_modules(self, only="."):
634
674
  """Return a filtered list of modules in the vortex package.
635
675
 
636
676
  :param str only: The regex used to filter the modules list.
@@ -638,22 +678,26 @@ class System(footprints.FootprintBase):
638
678
  if self.glove is not None:
639
679
  g = self.glove
640
680
  mfiles = [
641
- re.sub(r'^' + mroot + r'/', '', x)
642
- for mroot in (g.siteroot + '/src', g.siteroot + '/site')
681
+ re.sub(r"^" + mroot + r"/", "", x)
682
+ for mroot in (g.siteroot + "/src", g.siteroot + "/site")
643
683
  for x in self.ffind(mroot)
644
- if self.path.isfile(self.path.join(self.path.dirname(x), '__init__.py'))
684
+ if self.path.isfile(
685
+ self.path.join(self.path.dirname(x), "__init__.py")
686
+ )
645
687
  ]
646
688
  return [
647
- re.sub(r'(?:/__init__)?\.py$', '', x).replace('/', '.')
689
+ re.sub(r"(?:/__init__)?\.py$", "", x).replace("/", ".")
648
690
  for x in mfiles
649
- if (not x.startswith('.') and
650
- re.search(only, x, re.IGNORECASE) and
651
- x.endswith('.py'))
691
+ if (
692
+ not x.startswith(".")
693
+ and re.search(only, x, re.IGNORECASE)
694
+ and x.endswith(".py")
695
+ )
652
696
  ]
653
697
  else:
654
698
  raise RuntimeError("A glove must be defined")
655
699
 
656
- def vortex_loaded_modules(self, only='.', output=None):
700
+ def vortex_loaded_modules(self, only=".", output=None):
657
701
  """Check loaded modules, producing either a dump or a list of tuple (status, modulename).
658
702
 
659
703
  :param str only: The regex used to filter the modules list.
@@ -666,7 +710,7 @@ class System(footprints.FootprintBase):
666
710
  if not output:
667
711
  for m, s in checklist:
668
712
  print(str(s).ljust(8), m)
669
- print('--')
713
+ print("--")
670
714
  return True
671
715
  else:
672
716
  return checklist
@@ -674,13 +718,17 @@ class System(footprints.FootprintBase):
674
718
  def systems_reload(self):
675
719
  """Load extra systems modules not yet loaded."""
676
720
  extras = list()
677
- for modname in self.vortex_modules(only='systems'):
721
+ for modname in self.vortex_modules(only="systems"):
678
722
  if modname not in sys.modules:
679
723
  try:
680
724
  self.import_module(modname)
681
725
  extras.append(modname)
682
726
  except ValueError as err:
683
- logger.critical('systems_reload: cannot import module %s (%s)', modname, str(err))
727
+ logger.critical(
728
+ "systems_reload: cannot import module %s (%s)",
729
+ modname,
730
+ str(err),
731
+ )
684
732
  return extras
685
733
 
686
734
  # Redefinition of methods of the resource package...
@@ -692,16 +740,16 @@ class System(footprints.FootprintBase):
692
740
  """
693
741
  if type(r_id) is not int:
694
742
  r_id = r_id.upper()
695
- if not r_id.startswith('RLIMIT_'):
696
- r_id = 'RLIMIT_' + r_id
743
+ if not r_id.startswith("RLIMIT_"):
744
+ r_id = "RLIMIT_" + r_id
697
745
  r_id = getattr(self._rl, r_id, None)
698
746
  if r_id is None:
699
- raise ValueError('Invalid resource specified')
747
+ raise ValueError("Invalid resource specified")
700
748
  return r_id
701
749
 
702
750
  def setrlimit(self, r_id, r_limits):
703
751
  """Proxy to :mod:`resource` function of the same name."""
704
- self.stderr('setrlimit', r_id, r_limits)
752
+ self.stderr("setrlimit", r_id, r_limits)
705
753
  try:
706
754
  r_limits = tuple(r_limits)
707
755
  except TypeError:
@@ -710,14 +758,14 @@ class System(footprints.FootprintBase):
710
758
 
711
759
  def getrlimit(self, r_id):
712
760
  """Proxy to :mod:`resource` function of the same name."""
713
- self.stderr('getrlimit', r_id)
761
+ self.stderr("getrlimit", r_id)
714
762
  return self._rl.getrlimit(self.numrlimit(r_id))
715
763
 
716
764
  def getrusage(self, pid=None):
717
765
  """Proxy to :mod:`resource` function of the same name with current process as defaut."""
718
766
  if pid is None:
719
767
  pid = self._rl.RUSAGE_SELF
720
- self.stderr('getrusage', pid)
768
+ self.stderr("getrusage", pid)
721
769
  return self._rl.getrusage(pid)
722
770
 
723
771
  def import_module(self, modname):
@@ -728,12 +776,12 @@ class System(footprints.FootprintBase):
728
776
  def import_function(self, funcname):
729
777
  """Import the function named **funcname** qualified by a proper module name package."""
730
778
  thisfunc = None
731
- if '.' in funcname:
732
- thismod = self.import_module('.'.join(funcname.split('.')[:-1]))
779
+ if "." in funcname:
780
+ thismod = self.import_module(".".join(funcname.split(".")[:-1]))
733
781
  if thismod:
734
- thisfunc = getattr(thismod, funcname.split('.')[-1], None)
782
+ thisfunc = getattr(thismod, funcname.split(".")[-1], None)
735
783
  else:
736
- logger.error('Bad function path name <%s>' % funcname)
784
+ logger.error("Bad function path name <%s>" % funcname)
737
785
  return thisfunc
738
786
 
739
787
 
@@ -744,9 +792,7 @@ class OSExtended(System):
744
792
  """
745
793
 
746
794
  _abstract = True
747
- _footprint = dict(
748
- info = 'Abstract extended base system'
749
- )
795
+ _footprint = dict(info="Abstract extended base system")
750
796
 
751
797
  def __init__(self, *args, **kw):
752
798
  """
@@ -755,7 +801,7 @@ class OSExtended(System):
755
801
 
756
802
  * **rmtreemin** - as the minimal depth needed for a :meth:`rmsafe`.
757
803
  * **cmpaftercp** - as a boolean for activating full comparison after plain cp (default: *True*).
758
- * **ftraw** - allows ``smartft*`` methods to use the raw FTP commands
804
+ * **ftserv** - allows ``smartft*`` methods to use the raw FTP commands
759
805
  (e.g. ftget, ftput) instead of the internal Vortex's FTP client
760
806
  (default: *False*).
761
807
  * **ftputcmd** - The name of the raw FTP command for the "put" action
@@ -766,15 +812,15 @@ class OSExtended(System):
766
812
  (default: `FTP_FLAVOUR.CONNECTION_POOLS`). See the :meth:`ftp` method
767
813
  for more details.
768
814
  """
769
- logger.debug('Abstract System init %s', self.__class__)
770
- self._rmtreemin = kw.pop('rmtreemin', 3)
771
- self._cmpaftercp = kw.pop('cmpaftercp', True)
815
+ logger.debug("Abstract System init %s", self.__class__)
816
+ self._rmtreemin = kw.pop("rmtreemin", 3)
817
+ self._cmpaftercp = kw.pop("cmpaftercp", True)
772
818
  # Switches for rawft* methods
773
- self._ftraw = kw.pop('ftraw', None)
774
- self.ftputcmd = kw.pop('ftputcmd', None)
775
- self.ftgetcmd = kw.pop('ftgetcmd', None)
819
+ self._ftserv = kw.pop("ftserv", None)
820
+ self.ftputcmd = kw.pop("ftputcmd", None)
821
+ self.ftgetcmd = kw.pop("ftgetcmd", None)
776
822
  # FTP stuff again
777
- self.ftpflavour = kw.pop('ftpflavour', FTP_FLAVOUR.CONNECTION_POOLS)
823
+ self.ftpflavour = kw.pop("ftpflavour", FTP_FLAVOUR.CONNECTION_POOLS)
778
824
  self._current_ftppool = None
779
825
  # Some internal variables used by particular methods
780
826
  self._ftspool_cache = None
@@ -784,32 +830,42 @@ class OSExtended(System):
784
830
  # Go for the superclass' constructor
785
831
  super().__init__(*args, **kw)
786
832
  # Initialise possibly missing objects
787
- self.__dict__['_cpusinfo'] = None
788
- self.__dict__['_numainfo'] = None
789
- self.__dict__['_memoryinfo'] = None
790
- self.__dict__['_netstatsinfo'] = None
833
+ self.__dict__["_cpusinfo"] = None
834
+ self.__dict__["_numainfo"] = None
835
+ self.__dict__["_memoryinfo"] = None
836
+ self.__dict__["_netstatsinfo"] = None
791
837
 
792
838
  # Initialise the signal handler object
793
839
  self._signal_intercept_init()
794
840
 
795
841
  @property
796
- def ftraw(self):
842
+ def ftserv(self):
797
843
  """Use the system's FTP service (e.g. ftserv)."""
798
- if self._ftraw is None:
799
- return self.default_target.ftraw_default
844
+ if self._ftserv is None:
845
+ return self._use_ftserv()
800
846
  else:
801
- return self._ftraw
847
+ return self._ftserv
802
848
 
803
- @ftraw.setter
804
- def ftraw(self, value):
849
+ @ftserv.setter
850
+ def ftserv(self, value):
805
851
  """Use the system's FTP service (e.g. ftserv)."""
806
852
  self._ftraw = bool(value)
807
853
 
808
- @ftraw.deleter
809
- def ftraw(self):
854
+ @ftserv.deleter
855
+ def ftserv(self):
810
856
  """Use the system's FTP service (e.g. ftserv)."""
811
857
  self._ftraw = None
812
858
 
859
+ def _use_ftserv(self):
860
+ if not config.is_defined(section="ftserv"):
861
+ return False
862
+ for rgxp in config.from_config(
863
+ section="ftserv", key="hostname_patterns"
864
+ ):
865
+ if re.match(rgxp, self.hostname):
866
+ return True
867
+ return False
868
+
813
869
  def target(self, **kw):
814
870
  """
815
871
  Provide a default :class:`~vortex.tools.targets.Target` according
@@ -819,10 +875,7 @@ class OSExtended(System):
819
875
  * The object returned by this method will be used in subsequent calls
820
876
  to ::attr:`default_target` (this is the concept of frozen target).
821
877
  """
822
- desc = dict(
823
- hostname=self.hostname,
824
- sysname=self.sysname
825
- )
878
+ desc = dict(hostname=self.hostname, sysname=self.sysname)
826
879
  desc.update(kw)
827
880
  self._frozen_target = footprints.proxy.targets.default(**desc)
828
881
  return self._frozen_target
@@ -834,10 +887,18 @@ class OSExtended(System):
834
887
 
835
888
  def fmtspecific_mtd(self, method, fmt):
836
889
  """Check if a format specific implementation is available for a given format."""
837
- return hasattr(self, '{:s}_{:s}'.format(fmt, method))
838
-
839
- def popen(self, args, stdin=None, stdout=None, stderr=None, shell=False,
840
- output=False, bufsize=0): # @UnusedVariable
890
+ return hasattr(self, "{:s}_{:s}".format(fmt, method))
891
+
892
+ def popen(
893
+ self,
894
+ args,
895
+ stdin=None,
896
+ stdout=None,
897
+ stderr=None,
898
+ shell=False,
899
+ output=False,
900
+ bufsize=0,
901
+ ): # @UnusedVariable
841
902
  """Return an open pipe for the **args** command.
842
903
 
843
904
  :param str|list args: The command (+ its command-line arguments) to be
@@ -882,7 +943,14 @@ class OSExtended(System):
882
943
  stdin = subprocess.PIPE
883
944
  if stderr is True:
884
945
  stderr = subprocess.PIPE
885
- return subprocess.Popen(args, bufsize=bufsize, stdin=stdin, stdout=stdout, stderr=stderr, shell=shell)
946
+ return subprocess.Popen(
947
+ args,
948
+ bufsize=bufsize,
949
+ stdin=stdin,
950
+ stdout=stdout,
951
+ stderr=stderr,
952
+ shell=shell,
953
+ )
886
954
 
887
955
  def pclose(self, p, ok=None):
888
956
  """Do its best to nicely shutdown the process started by **p**.
@@ -905,7 +973,7 @@ class OSExtended(System):
905
973
  p.terminate()
906
974
  except OSError as e:
907
975
  if e.errno == 3:
908
- logger.debug('Processus %s alreaded terminated.' % str(p))
976
+ logger.debug("Processus %s alreaded terminated." % str(p))
909
977
  else:
910
978
  raise
911
979
 
@@ -917,9 +985,21 @@ class OSExtended(System):
917
985
  else:
918
986
  return False
919
987
 
920
- def spawn(self, args, ok=None, shell=False, stdin=None, output=None,
921
- outmode='a+b', outsplit=True, silent=False, fatal=True,
922
- taskset=None, taskset_id=0, taskset_bsize=1):
988
+ def spawn(
989
+ self,
990
+ args,
991
+ ok=None,
992
+ shell=False,
993
+ stdin=None,
994
+ output=None,
995
+ outmode="a+b",
996
+ outsplit=True,
997
+ silent=False,
998
+ fatal=True,
999
+ taskset=None,
1000
+ taskset_id=0,
1001
+ taskset_bsize=1,
1002
+ ):
923
1003
  """Subprocess call of **args**.
924
1004
 
925
1005
  :param str|list[str] args: The command (+ its command-line arguments) to be
@@ -986,25 +1066,25 @@ class OSExtended(System):
986
1066
  stdin = subprocess.PIPE
987
1067
  localenv = self._os.environ.copy()
988
1068
  if taskset is not None:
989
- taskset_def = taskset.split('_')
990
- taskset, taskset_cmd, taskset_env = self.cpus_affinity_get(taskset_id,
991
- taskset_bsize,
992
- *taskset_def)
1069
+ taskset_def = taskset.split("_")
1070
+ taskset, taskset_cmd, taskset_env = self.cpus_affinity_get(
1071
+ taskset_id, taskset_bsize, *taskset_def
1072
+ )
993
1073
  if taskset:
994
1074
  localenv.update(taskset_env)
995
1075
  else:
996
1076
  logger.warning("CPU binding is not available on this platform")
997
1077
  if isinstance(args, str):
998
1078
  if taskset:
999
- args = taskset_cmd + ' ' + args
1079
+ args = taskset_cmd + " " + args
1000
1080
  if self.timer:
1001
- args = 'time ' + args
1081
+ args = "time " + args
1002
1082
  self.stderr(args)
1003
1083
  else:
1004
1084
  if taskset:
1005
1085
  args[:0] = taskset_cmd
1006
1086
  if self.timer:
1007
- args[:0] = ['time']
1087
+ args[:0] = ["time"]
1008
1088
  self.stderr(*args)
1009
1089
  if isinstance(output, bool):
1010
1090
  if output:
@@ -1017,36 +1097,47 @@ class OSExtended(System):
1017
1097
  cmdout, cmderr = output, output
1018
1098
  p = None
1019
1099
  try:
1020
- p = subprocess.Popen(args, stdin=stdin, stdout=cmdout, stderr=cmderr,
1021
- shell=shell, env=localenv)
1100
+ p = subprocess.Popen(
1101
+ args,
1102
+ stdin=stdin,
1103
+ stdout=cmdout,
1104
+ stderr=cmderr,
1105
+ shell=shell,
1106
+ env=localenv,
1107
+ )
1022
1108
  p_out, p_err = p.communicate()
1023
1109
  except ValueError as e:
1024
1110
  logger.critical(
1025
- 'Weird arguments to Popen ({!s}, stdout={!s}, stderr={!s}, shell={!s})'.format(
1111
+ "Weird arguments to Popen ({!s}, stdout={!s}, stderr={!s}, shell={!s})".format(
1026
1112
  args, cmdout, cmderr, shell
1027
1113
  )
1028
1114
  )
1029
- logger.critical('Caught exception: %s', e)
1115
+ logger.critical("Caught exception: %s", e)
1030
1116
  if fatal:
1031
1117
  raise
1032
1118
  else:
1033
- logger.warning('Carry on because fatal is off')
1119
+ logger.warning("Carry on because fatal is off")
1034
1120
  except OSError:
1035
- logger.critical('Could not call %s', str(args))
1121
+ logger.critical("Could not call %s", str(args))
1036
1122
  if fatal:
1037
1123
  raise
1038
1124
  else:
1039
- logger.warning('Carry on because fatal is off')
1125
+ logger.warning("Carry on because fatal is off")
1040
1126
  except Exception as perr:
1041
- logger.critical('System returns %s', str(perr))
1127
+ logger.critical("System returns %s", str(perr))
1042
1128
  if fatal:
1043
- raise RuntimeError('System {!s} spawned {!s} got [{!s}]: {!s}'
1044
- .format(self, args, p.returncode, perr))
1129
+ raise RuntimeError(
1130
+ "System {!s} spawned {!s} got [{!s}]: {!s}".format(
1131
+ self, args, p.returncode, perr
1132
+ )
1133
+ )
1045
1134
  else:
1046
- logger.warning('Carry on because fatal is off')
1135
+ logger.warning("Carry on because fatal is off")
1047
1136
  except (SignalInterruptError, KeyboardInterrupt) as perr:
1048
- logger.critical('The python process was killed: %s. Trying to terminate the subprocess.',
1049
- str(perr))
1137
+ logger.critical(
1138
+ "The python process was killed: %s. Trying to terminate the subprocess.",
1139
+ str(perr),
1140
+ )
1050
1141
  if p:
1051
1142
  if shell:
1052
1143
  # Kill the process group: apparently it's the only way when shell=T
@@ -1056,24 +1147,26 @@ class OSExtended(System):
1056
1147
  p.wait()
1057
1148
  raise # Fatal has no effect on that !
1058
1149
  else:
1059
- plocale = locale.getlocale()[1] or 'ascii'
1150
+ plocale = locale.getlocale()[1] or "ascii"
1060
1151
  if p.returncode in ok:
1061
1152
  if isinstance(output, bool) and output:
1062
- rc = p_out.decode(plocale, 'replace')
1153
+ rc = p_out.decode(plocale, "replace")
1063
1154
  if outsplit:
1064
- rc = rc.rstrip('\n').split('\n')
1155
+ rc = rc.rstrip("\n").split("\n")
1065
1156
  p.stdout.close()
1066
1157
  else:
1067
1158
  rc = not bool(p.returncode)
1068
1159
  else:
1069
1160
  if not silent:
1070
- logger.warning('Bad return code [%d] for %s', p.returncode, str(args))
1161
+ logger.warning(
1162
+ "Bad return code [%d] for %s", p.returncode, str(args)
1163
+ )
1071
1164
  if isinstance(output, bool) and output:
1072
- sys.stderr.write(p_err.decode(plocale, 'replace'))
1165
+ sys.stderr.write(p_err.decode(plocale, "replace"))
1073
1166
  if fatal:
1074
1167
  raise ExecutionError()
1075
1168
  else:
1076
- logger.warning('Carry on because fatal is off')
1169
+ logger.warning("Carry on because fatal is off")
1077
1170
  finally:
1078
1171
  self._rclast = p.returncode if p else 1
1079
1172
  if isinstance(output, bool) and p:
@@ -1107,11 +1200,11 @@ class OSExtended(System):
1107
1200
  """Current working directory."""
1108
1201
  if output is None:
1109
1202
  output = self.output
1110
- self.stderr('pwd')
1203
+ self.stderr("pwd")
1111
1204
  try:
1112
1205
  realpwd = self._os.getcwd()
1113
1206
  except OSError as e:
1114
- logger.error('getcwdu failed: %s.', str(e))
1207
+ logger.error("getcwdu failed: %s.", str(e))
1115
1208
  return None
1116
1209
  if output:
1117
1210
  return realpwd
@@ -1122,7 +1215,7 @@ class OSExtended(System):
1122
1215
  def cd(self, pathtogo, create=False):
1123
1216
  """Change the current working directory to **pathtogo**."""
1124
1217
  pathtogo = self.path.expanduser(pathtogo)
1125
- self.stderr('cd', pathtogo, create)
1218
+ self.stderr("cd", pathtogo, create)
1126
1219
  if create:
1127
1220
  self.mkdir(pathtogo)
1128
1221
  self._os.chdir(pathtogo)
@@ -1143,11 +1236,13 @@ class OSExtended(System):
1143
1236
  :param prefix: The temporary directory name will start with that suffix
1144
1237
  :param dir: The temporary directory will be created in that directory
1145
1238
  """
1146
- self.stderr('temporary_dir_context starts', suffix)
1147
- self.stderr('tempfile.TemporaryDirectory', suffix, prefix, dir)
1148
- with tempfile.TemporaryDirectory(suffix=suffix, prefix=prefix, dir=dir) as tmp_dir:
1239
+ self.stderr("temporary_dir_context starts", suffix)
1240
+ self.stderr("tempfile.TemporaryDirectory", suffix, prefix, dir)
1241
+ with tempfile.TemporaryDirectory(
1242
+ suffix=suffix, prefix=prefix, dir=dir
1243
+ ) as tmp_dir:
1149
1244
  yield tmp_dir
1150
- self.stderr('tempfile.TemporaryDirectory cleanup', tmp_dir)
1245
+ self.stderr("tempfile.TemporaryDirectory cleanup", tmp_dir)
1151
1246
 
1152
1247
  @contextlib.contextmanager
1153
1248
  def temporary_dir_cdcontext(self, suffix=None, prefix=None, dir=None):
@@ -1155,23 +1250,27 @@ class OSExtended(System):
1155
1250
 
1156
1251
  For a description of the context's arguments, see :func:`temporary_dir_context`.
1157
1252
  """
1158
- with self.temporary_dir_context(suffix=suffix, prefix=prefix, dir=dir) as tmp_dir:
1253
+ with self.temporary_dir_context(
1254
+ suffix=suffix, prefix=prefix, dir=dir
1255
+ ) as tmp_dir:
1159
1256
  with self.cdcontext(tmp_dir, create=False, clean_onexit=False):
1160
1257
  yield tmp_dir
1161
1258
 
1162
1259
  def ffind(self, *args):
1163
1260
  """Recursive file find. Arguments are starting paths."""
1164
1261
  if not args:
1165
- args = ['*']
1262
+ args = ["*"]
1166
1263
  else:
1167
1264
  args = [self.path.expanduser(x) for x in args]
1168
1265
  files = []
1169
- self.stderr('ffind', *args)
1266
+ self.stderr("ffind", *args)
1170
1267
  for pathtogo in self.glob(*args):
1171
1268
  if self.path.isfile(pathtogo):
1172
1269
  files.append(pathtogo)
1173
1270
  else:
1174
- for root, u_dirs, filenames in self._os.walk(pathtogo): # @UnusedVariable
1271
+ for root, u_dirs, filenames in self._os.walk(
1272
+ pathtogo
1273
+ ): # @UnusedVariable
1175
1274
  files.extend([self.path.join(root, f) for f in filenames])
1176
1275
  return sorted(files)
1177
1276
 
@@ -1184,8 +1283,13 @@ class OSExtended(System):
1184
1283
  if self._os.path.exists(filename):
1185
1284
  is_x = bool(self._os.stat(filename).st_mode & 1)
1186
1285
  if not is_x and force:
1187
- self.chmod(filename,
1188
- self._os.stat(filename).st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
1286
+ self.chmod(
1287
+ filename,
1288
+ self._os.stat(filename).st_mode
1289
+ | stat.S_IXUSR
1290
+ | stat.S_IXGRP
1291
+ | stat.S_IXOTH,
1292
+ )
1189
1293
  is_x = True
1190
1294
  return is_x
1191
1295
  else:
@@ -1199,9 +1303,16 @@ class OSExtended(System):
1199
1303
  """
1200
1304
  if self._os.path.exists(filename):
1201
1305
  mode = self._os.stat(filename).st_mode
1202
- is_r = all([bool(mode & i) for i in [stat.S_IRUSR, stat.S_IRGRP, stat.S_IROTH]])
1306
+ is_r = all(
1307
+ [
1308
+ bool(mode & i)
1309
+ for i in [stat.S_IRUSR, stat.S_IRGRP, stat.S_IROTH]
1310
+ ]
1311
+ )
1203
1312
  if not is_r and force:
1204
- self.chmod(filename, mode | stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH)
1313
+ self.chmod(
1314
+ filename, mode | stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH
1315
+ )
1205
1316
  is_r = True
1206
1317
  return is_r
1207
1318
  else:
@@ -1238,7 +1349,7 @@ class OSExtended(System):
1238
1349
  def readonly(self, inodename):
1239
1350
  """Set permissions of the ``inodename`` object to read-only."""
1240
1351
  inodename = self.path.expanduser(inodename)
1241
- self.stderr('readonly', inodename)
1352
+ self.stderr("readonly", inodename)
1242
1353
  rc = None
1243
1354
  if self._os.path.exists(inodename):
1244
1355
  if self._os.path.isdir(inodename):
@@ -1246,7 +1357,10 @@ class OSExtended(System):
1246
1357
  else:
1247
1358
  st = self.stat(inodename).st_mode
1248
1359
  if st & stat.S_IWUSR or st & stat.S_IWGRP or st & stat.S_IWOTH:
1249
- rc = self.chmod(inodename, st & ~(stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH))
1360
+ rc = self.chmod(
1361
+ inodename,
1362
+ st & ~(stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH),
1363
+ )
1250
1364
  else:
1251
1365
  rc = True
1252
1366
  return rc
@@ -1266,7 +1380,7 @@ class OSExtended(System):
1266
1380
  def touch(self, filename):
1267
1381
  """Clone of the eponymous unix command."""
1268
1382
  filename = self.path.expanduser(filename)
1269
- self.stderr('touch', filename)
1383
+ self.stderr("touch", filename)
1270
1384
  rc = True
1271
1385
  if self.path.exists(filename):
1272
1386
  # Note: "filename" might as well be a directory...
@@ -1275,7 +1389,7 @@ class OSExtended(System):
1275
1389
  except Exception:
1276
1390
  rc = False
1277
1391
  else:
1278
- fh = open(filename, 'a')
1392
+ fh = open(filename, "a")
1279
1393
  fh.close()
1280
1394
  return rc
1281
1395
 
@@ -1287,13 +1401,13 @@ class OSExtended(System):
1287
1401
  """
1288
1402
  objpath = self.path.expanduser(objpath)
1289
1403
  if self._os.path.exists(objpath):
1290
- self.stderr('remove', objpath)
1404
+ self.stderr("remove", objpath)
1291
1405
  if self._os.path.isdir(objpath):
1292
1406
  self.rmtree(objpath)
1293
1407
  else:
1294
1408
  self.unlink(objpath)
1295
1409
  else:
1296
- self.stderr('clear', objpath)
1410
+ self.stderr("clear", objpath)
1297
1411
  return not self._os.path.exists(objpath)
1298
1412
 
1299
1413
  @fmtshcmd
@@ -1311,35 +1425,48 @@ class OSExtended(System):
1311
1425
  expressions are used).
1312
1426
  """
1313
1427
  if not pscmd:
1314
- pscmd = ['ps']
1428
+ pscmd = ["ps"]
1315
1429
  if opts is None:
1316
1430
  opts = []
1317
1431
  pscmd.extend(self._psopts)
1318
1432
  pscmd.extend(opts)
1319
1433
  self.stderr(*pscmd)
1320
- psall = subprocess.Popen(pscmd, stdout=subprocess.PIPE).communicate()[0].split('\n')
1434
+ psall = (
1435
+ subprocess.Popen(pscmd, stdout=subprocess.PIPE)
1436
+ .communicate()[0]
1437
+ .split("\n")
1438
+ )
1321
1439
  if search:
1322
1440
  psall = filter(lambda x: re.search(search, x), psall)
1323
1441
  return [x.strip() for x in psall]
1324
1442
 
1325
1443
  def sleep(self, nbsecs):
1326
1444
  """Clone of the unix eponymous command."""
1327
- self.stderr('sleep', nbsecs)
1445
+ self.stderr("sleep", nbsecs)
1328
1446
  time.sleep(nbsecs)
1329
1447
 
1330
1448
  def setulimit(self, r_id):
1331
1449
  """Set an unlimited value to the specified resource (**r_id**)."""
1332
- self.stderr('setulimit', r_id)
1450
+ self.stderr("setulimit", r_id)
1333
1451
  u_soft, hard = self.getrlimit(r_id) # @UnusedVariable
1334
1452
  if hard != self._rl.RLIM_INFINITY:
1335
- logger.info('Unable to raise the %s soft limit to "unlimited", ' +
1336
- 'using the hard limit instead (%s).', str(r_id), str(hard))
1453
+ logger.info(
1454
+ 'Unable to raise the %s soft limit to "unlimited", '
1455
+ + "using the hard limit instead (%s).",
1456
+ str(r_id),
1457
+ str(hard),
1458
+ )
1337
1459
  return self.setrlimit(r_id, (hard, hard))
1338
1460
 
1339
1461
  def ulimit(self):
1340
1462
  """Dump the user limits currently defined."""
1341
- for limit in [r for r in dir(self._rl) if r.startswith('RLIMIT_')]:
1342
- print(' ', limit.ljust(16), ':', self._rl.getrlimit(getattr(self._rl, limit)))
1463
+ for limit in [r for r in dir(self._rl) if r.startswith("RLIMIT_")]:
1464
+ print(
1465
+ " ",
1466
+ limit.ljust(16),
1467
+ ":",
1468
+ self._rl.getrlimit(getattr(self._rl, limit)),
1469
+ )
1343
1470
 
1344
1471
  @property
1345
1472
  def cpus_info(self):
@@ -1352,7 +1479,9 @@ class OSExtended(System):
1352
1479
  """
1353
1480
  return self._cpusinfo
1354
1481
 
1355
- def cpus_ids_per_blocks(self, blocksize=1, topology='raw', hexmask=False): # @UnusedVariable
1482
+ def cpus_ids_per_blocks(
1483
+ self, blocksize=1, topology="raw", hexmask=False
1484
+ ): # @UnusedVariable
1356
1485
  """Get the list of CPUs IDs ordered for subsequent binding.
1357
1486
 
1358
1487
  :param int blocksize: The number of thread consumed by one task
@@ -1361,14 +1490,16 @@ class OSExtended(System):
1361
1490
  """
1362
1491
  return []
1363
1492
 
1364
- def cpus_ids_dispenser(self, topology='raw'):
1493
+ def cpus_ids_dispenser(self, topology="raw"):
1365
1494
  """Get a dispenser of CPUs IDs for nicely ordered for subsequent binding.
1366
1495
 
1367
1496
  :param str topology: The task distribution scheme
1368
1497
  """
1369
1498
  return None
1370
1499
 
1371
- def cpus_affinity_get(self, taskid, blocksize=1, method='default', topology='raw'): # @UnusedVariable
1500
+ def cpus_affinity_get(
1501
+ self, taskid, blocksize=1, method="default", topology="raw"
1502
+ ): # @UnusedVariable
1372
1503
  """Get the necessary command/environment to set the CPUs affinity.
1373
1504
 
1374
1505
  :param int taskid: the task number
@@ -1421,7 +1552,9 @@ class OSExtended(System):
1421
1552
  netstat may not be implemented.
1422
1553
  """
1423
1554
  if self.netstatsinfo is None:
1424
- raise NotImplementedError('This function is not implemented on this system.')
1555
+ raise NotImplementedError(
1556
+ "This function is not implemented on this system."
1557
+ )
1425
1558
  return self.netstatsinfo.available_localport()
1426
1559
 
1427
1560
  def check_localport(self, port):
@@ -1431,12 +1564,14 @@ class OSExtended(System):
1431
1564
  netstat may not be implemented.
1432
1565
  """
1433
1566
  if self.netstatsinfo is None:
1434
- raise NotImplementedError('This function is not implemented on this system.')
1567
+ raise NotImplementedError(
1568
+ "This function is not implemented on this system."
1569
+ )
1435
1570
  return self.netstatsinfo.check_localport(port)
1436
1571
 
1437
1572
  def clear(self):
1438
1573
  """Clone of the unix eponymous command."""
1439
- self._os.system('clear')
1574
+ self._os.system("clear")
1440
1575
 
1441
1576
  @property
1442
1577
  def cls(self):
@@ -1444,8 +1579,14 @@ class OSExtended(System):
1444
1579
  self.clear()
1445
1580
  return None
1446
1581
 
1447
- def rawopts(self, cmdline=None, defaults=None,
1448
- isnone=isnonedef, istrue=istruedef, isfalse=isfalsedef):
1582
+ def rawopts(
1583
+ self,
1584
+ cmdline=None,
1585
+ defaults=None,
1586
+ isnone=isnonedef,
1587
+ istrue=istruedef,
1588
+ isfalse=isfalsedef,
1589
+ ):
1449
1590
  """Parse a simple options command line that looks like `` key=value``.
1450
1591
 
1451
1592
  :param str cmdline: The command line to be processed (if *None*, ``sys.argv``
@@ -1461,11 +1602,13 @@ class OSExtended(System):
1461
1602
  try:
1462
1603
  opts.update(defaults)
1463
1604
  except (ValueError, TypeError):
1464
- logger.warning('Could not update options default: %s', defaults)
1605
+ logger.warning(
1606
+ "Could not update options default: %s", defaults
1607
+ )
1465
1608
 
1466
1609
  if cmdline is None:
1467
1610
  cmdline = sys.argv[1:]
1468
- opts.update(dict([x.split('=') for x in cmdline]))
1611
+ opts.update(dict([x.split("=") for x in cmdline]))
1469
1612
  for k, v in opts.items():
1470
1613
  if v not in (None, True, False):
1471
1614
  if istrue.match(v):
@@ -1479,9 +1622,10 @@ class OSExtended(System):
1479
1622
  def is_iofile(self, iocandidate):
1480
1623
  """Check if actual **iocandidate** is a valid filename or io stream."""
1481
1624
  return iocandidate is not None and (
1482
- (isinstance(iocandidate, str) and self.path.exists(iocandidate)) or
1483
- isinstance(iocandidate, io.IOBase) or
1484
- isinstance(iocandidate, io.BytesIO) or isinstance(iocandidate, io.StringIO)
1625
+ (isinstance(iocandidate, str) and self.path.exists(iocandidate))
1626
+ or isinstance(iocandidate, io.IOBase)
1627
+ or isinstance(iocandidate, io.BytesIO)
1628
+ or isinstance(iocandidate, io.StringIO)
1485
1629
  )
1486
1630
 
1487
1631
  @contextlib.contextmanager
@@ -1512,20 +1656,28 @@ class OSExtended(System):
1512
1656
  hostname = self.glove.default_fthost
1513
1657
  if not hostname:
1514
1658
  if fatal:
1515
- raise ValueError('An *hostname* must be provided one way or another')
1659
+ raise ValueError(
1660
+ "An *hostname* must be provided one way or another"
1661
+ )
1516
1662
  return hostname
1517
1663
 
1518
1664
  def fix_ftuser(self, hostname, logname, fatal=True, defaults_to_user=True):
1519
1665
  """Given *hostname*, if *logname* is None, tries to find a default value for it."""
1520
1666
  if logname is None:
1521
1667
  if self.glove is not None:
1522
- logname = self.glove.getftuser(hostname, defaults_to_user=defaults_to_user)
1668
+ logname = self.glove.getftuser(
1669
+ hostname, defaults_to_user=defaults_to_user
1670
+ )
1523
1671
  else:
1524
1672
  if fatal:
1525
- raise ValueError("Either a *logname* or a glove must be set-up")
1673
+ raise ValueError(
1674
+ "Either a *logname* or a glove must be set-up"
1675
+ )
1526
1676
  return logname
1527
1677
 
1528
- def ftp(self, hostname, logname=None, delayed=False, port=DEFAULT_FTP_PORT):
1678
+ def ftp(
1679
+ self, hostname, logname=None, delayed=False, port=DEFAULT_FTP_PORT
1680
+ ):
1529
1681
  """Return an FTP client object.
1530
1682
 
1531
1683
  :param str hostname: the remote host's name for FTP.
@@ -1555,20 +1707,36 @@ class OSExtended(System):
1555
1707
  logname = self.fix_ftuser(hostname, logname)
1556
1708
  if port is None:
1557
1709
  port = DEFAULT_FTP_PORT
1558
- if self.ftpflavour == FTP_FLAVOUR.CONNECTION_POOLS and self._current_ftppool is not None:
1559
- return self._current_ftppool.deal(hostname, logname, port=port, delayed=delayed)
1710
+ if (
1711
+ self.ftpflavour == FTP_FLAVOUR.CONNECTION_POOLS
1712
+ and self._current_ftppool is not None
1713
+ ):
1714
+ return self._current_ftppool.deal(
1715
+ hostname, logname, port=port, delayed=delayed
1716
+ )
1560
1717
  else:
1561
- ftpclass = AutoRetriesFtp if self.ftpflavour != FTP_FLAVOUR.STD else StdFtp
1718
+ ftpclass = (
1719
+ AutoRetriesFtp
1720
+ if self.ftpflavour != FTP_FLAVOUR.STD
1721
+ else StdFtp
1722
+ )
1562
1723
  ftpbox = ftpclass(self, hostname, port=port)
1563
1724
  rc = ftpbox.fastlogin(logname, delayed=delayed)
1564
1725
  if rc:
1565
1726
  return ftpbox
1566
1727
  else:
1567
- logger.warning('Could not login on %s as %s [%s]', hostname, logname, str(rc))
1728
+ logger.warning(
1729
+ "Could not login on %s as %s [%s]",
1730
+ hostname,
1731
+ logname,
1732
+ str(rc),
1733
+ )
1568
1734
  return None
1569
1735
 
1570
1736
  @contextlib.contextmanager
1571
- def ftpcontext(self, hostname, logname=None, delayed=False, port=DEFAULT_FTP_PORT):
1737
+ def ftpcontext(
1738
+ self, hostname, logname=None, delayed=False, port=DEFAULT_FTP_PORT
1739
+ ):
1572
1740
  """Create an FTP object and close it when the context exits.
1573
1741
 
1574
1742
  For a description of the context's arguments, see :func:`ftp`.
@@ -1591,8 +1759,15 @@ class OSExtended(System):
1591
1759
  return remote
1592
1760
 
1593
1761
  @fmtshcmd
1594
- def ftget(self, source, destination, hostname=None, logname=None, port=DEFAULT_FTP_PORT,
1595
- cpipeline=None):
1762
+ def ftget(
1763
+ self,
1764
+ source,
1765
+ destination,
1766
+ hostname=None,
1767
+ logname=None,
1768
+ port=DEFAULT_FTP_PORT,
1769
+ cpipeline=None,
1770
+ ):
1596
1771
  """Proceed to a direct ftp get on the specified target (using Vortex's FTP client).
1597
1772
 
1598
1773
  :param str source: the remote path to get data
@@ -1616,10 +1791,12 @@ class OSExtended(System):
1616
1791
  if cpipeline is None:
1617
1792
  rc = ftp.get(source, destination)
1618
1793
  else:
1619
- with cpipeline.stream2uncompress(destination) as cdestination:
1794
+ with cpipeline.stream2uncompress(
1795
+ destination
1796
+ ) as cdestination:
1620
1797
  rc = ftp.get(source, cdestination)
1621
1798
  except ftplib.all_errors as e:
1622
- logger.warning('An FTP error occured: %s', str(e))
1799
+ logger.warning("An FTP error occured: %s", str(e))
1623
1800
  rc = False
1624
1801
  finally:
1625
1802
  ftp.close()
@@ -1628,8 +1805,16 @@ class OSExtended(System):
1628
1805
  return False
1629
1806
 
1630
1807
  @fmtshcmd
1631
- def ftput(self, source, destination, hostname=None, logname=None, port=DEFAULT_FTP_PORT,
1632
- cpipeline=None, sync=False): # @UnusedVariable
1808
+ def ftput(
1809
+ self,
1810
+ source,
1811
+ destination,
1812
+ hostname=None,
1813
+ logname=None,
1814
+ port=DEFAULT_FTP_PORT,
1815
+ cpipeline=None,
1816
+ sync=False,
1817
+ ): # @UnusedVariable
1633
1818
  """Proceed to a direct ftp put on the specified target (using Vortex's FTP client).
1634
1819
 
1635
1820
  :param source: The source of data (either a path to file or a
@@ -1655,33 +1840,45 @@ class OSExtended(System):
1655
1840
  if cpipeline is None:
1656
1841
  rc = ftp.put(source, destination)
1657
1842
  else:
1658
- with cpipeline.compress2stream(source, iosponge=True) as csource:
1843
+ with cpipeline.compress2stream(
1844
+ source, iosponge=True
1845
+ ) as csource:
1659
1846
  # csource is an IoSponge consequently the size attribute exists
1660
- rc = ftp.put(csource, destination, size=csource.size)
1847
+ rc = ftp.put(
1848
+ csource, destination, size=csource.size
1849
+ )
1661
1850
  except ftplib.all_errors as e:
1662
- logger.warning('An FTP error occured: %s', str(e))
1851
+ logger.warning("An FTP error occured: %s", str(e))
1663
1852
  rc = False
1664
1853
  finally:
1665
1854
  ftp.close()
1666
1855
  else:
1667
- raise OSError('No such file or directory: {!r}'.format(source))
1856
+ raise OSError("No such file or directory: {!r}".format(source))
1668
1857
  return rc
1669
1858
 
1670
1859
  def ftspool_cache(self):
1671
1860
  """Return a cache object for the FtSpool."""
1672
- if self._ftspool_cache is None:
1673
- self._ftspool_cache = footprints.proxy.cache(kind='ftstash')
1861
+ if self._ftspool_cache is not None:
1862
+ return self._ftspool_cache
1863
+ self._ftspool_cache = footprints.proxy.cache(
1864
+ entry=os.path.join(
1865
+ vortex.data.stores.get_cache_location(), "ftspool"
1866
+ ),
1867
+ )
1674
1868
  return self._ftspool_cache
1675
1869
 
1676
1870
  def copy2ftspool(self, source, nest=False, **kwargs):
1677
1871
  """Make a copy of **source** to the FtSpool cache."""
1678
- h = hashlib.new('md5')
1679
- h.update(source.encode(encoding='utf-8'))
1680
- outputname = 'vortex_{:s}_P{:06d}_{:s}'.format(date.now().strftime('%Y%m%d%H%M%S-%f'),
1681
- self.getpid(), h.hexdigest())
1872
+ h = hashlib.new("md5")
1873
+ h.update(source.encode(encoding="utf-8"))
1874
+ outputname = "vortex_{:s}_P{:06d}_{:s}".format(
1875
+ date.now().strftime("%Y%m%d%H%M%S-%f"),
1876
+ self.getpid(),
1877
+ h.hexdigest(),
1878
+ )
1682
1879
  if nest:
1683
1880
  outputname = self.path.join(outputname, self.path.basename(source))
1684
- kwargs['intent'] = 'in' # Force intent=in
1881
+ kwargs["intent"] = "in" # Force intent=in
1685
1882
  if self.ftspool_cache().insert(outputname, source, **kwargs):
1686
1883
  return self.ftspool_cache().fullpath(outputname)
1687
1884
  else:
@@ -1691,42 +1888,69 @@ class OSExtended(System):
1691
1888
  """Given **source** and **destination**, is FtServ usable ?"""
1692
1889
  return isinstance(source, str) and isinstance(destination, str)
1693
1890
 
1694
- def ftserv_put(self, source, destination, hostname=None, logname=None, port=None,
1695
- specialshell=None, sync=False):
1891
+ def ftserv_put(
1892
+ self,
1893
+ source,
1894
+ destination,
1895
+ hostname=None,
1896
+ logname=None,
1897
+ port=None,
1898
+ specialshell=None,
1899
+ sync=False,
1900
+ ):
1696
1901
  """Asynchronous put of a file using FtServ."""
1697
1902
  if self.ftserv_allowed(source, destination):
1698
1903
  if self.path.exists(source):
1699
- ftcmd = self.ftputcmd or 'ftput'
1904
+ ftcmd = self.ftputcmd or "ftput"
1700
1905
  hostname = self.fix_fthostname(hostname, fatal=False)
1701
1906
  logname = self.fix_ftuser(hostname, logname, fatal=False)
1702
1907
  extras = list()
1703
1908
  if not sync:
1704
- extras.extend(['-q', ])
1909
+ extras.extend(
1910
+ [
1911
+ "-q",
1912
+ ]
1913
+ )
1705
1914
  if hostname:
1706
1915
  if port is not None:
1707
- hostname += ':{:s}'.format(port)
1708
- extras.extend(['-h', hostname])
1916
+ hostname += ":{:s}".format(port)
1917
+ extras.extend(["-h", hostname])
1709
1918
  if logname:
1710
- extras.extend(['-u', logname])
1919
+ extras.extend(["-u", logname])
1711
1920
  if specialshell:
1712
- extras.extend(['-s', specialshell])
1921
+ extras.extend(["-s", specialshell])
1713
1922
  # Remove ~/ and ~logname/ from the destinations' path
1714
- actual_dest = re.sub('^~/+', '', destination)
1923
+ actual_dest = re.sub("^~/+", "", destination)
1715
1924
  if logname:
1716
- actual_dest = re.sub('^~{:s}/+'.format(logname), '', actual_dest)
1925
+ actual_dest = re.sub(
1926
+ "^~{:s}/+".format(logname), "", actual_dest
1927
+ )
1717
1928
  try:
1718
- rc = self.spawn([ftcmd,
1719
- '-o', 'mkdir', ] + # Automatically create subdirectories
1720
- extras + [source, actual_dest], output=False)
1929
+ rc = self.spawn(
1930
+ [
1931
+ ftcmd,
1932
+ "-o",
1933
+ "mkdir",
1934
+ ] # Automatically create subdirectories
1935
+ + extras
1936
+ + [source, actual_dest],
1937
+ output=False,
1938
+ )
1721
1939
  except ExecutionError:
1722
1940
  rc = False
1723
1941
  else:
1724
- raise OSError('No such file or directory: {!s}'.format(source))
1942
+ raise OSError("No such file or directory: {!s}".format(source))
1725
1943
  else:
1726
- raise OSError('Source or destination is not a plain file path: {!r}'.format(source))
1944
+ raise OSError(
1945
+ "Source or destination is not a plain file path: {!r}".format(
1946
+ source
1947
+ )
1948
+ )
1727
1949
  return rc
1728
1950
 
1729
- def ftserv_get(self, source, destination, hostname=None, logname=None, port=None):
1951
+ def ftserv_get(
1952
+ self, source, destination, hostname=None, logname=None, port=None
1953
+ ):
1730
1954
  """Get a file using FtServ."""
1731
1955
  if self.ftserv_allowed(source, destination):
1732
1956
  if self.filecocoon(destination):
@@ -1736,67 +1960,103 @@ class OSExtended(System):
1736
1960
  extras = list()
1737
1961
  if hostname:
1738
1962
  if port is not None:
1739
- hostname += ':{:s}'.format(port)
1740
- extras.extend(['-h', hostname])
1963
+ hostname += ":{:s}".format(port)
1964
+ extras.extend(["-h", hostname])
1741
1965
  if logname:
1742
- extras.extend(['-u', logname])
1743
- ftcmd = self.ftgetcmd or 'ftget'
1966
+ extras.extend(["-u", logname])
1967
+ ftcmd = self.ftgetcmd or "ftget"
1744
1968
  try:
1745
- rc = self.spawn([ftcmd, ] + extras + [source, destination], output=False)
1969
+ rc = self.spawn(
1970
+ [
1971
+ ftcmd,
1972
+ ]
1973
+ + extras
1974
+ + [source, destination],
1975
+ output=False,
1976
+ )
1746
1977
  except ExecutionError:
1747
1978
  rc = False
1748
1979
  else:
1749
- raise OSError('Could not cocoon: {!s}'.format(destination))
1980
+ raise OSError("Could not cocoon: {!s}".format(destination))
1750
1981
  else:
1751
- raise OSError('Source or destination is not a plain file path: {!r}'.format(source))
1982
+ raise OSError(
1983
+ "Source or destination is not a plain file path: {!r}".format(
1984
+ source
1985
+ )
1986
+ )
1752
1987
  return rc
1753
1988
 
1754
- def ftserv_batchget(self, source, destination, hostname=None, logname=None, port=None):
1989
+ def ftserv_batchget(
1990
+ self, source, destination, hostname=None, logname=None, port=None
1991
+ ):
1755
1992
  """Get a list of files using FtServ.
1756
1993
 
1757
1994
  :note: **source** and **destination** are list or tuple.
1758
1995
  """
1759
- if all([self.ftserv_allowed(s, d) for s, d in zip(source, destination)]):
1996
+ if all(
1997
+ [self.ftserv_allowed(s, d) for s, d in zip(source, destination)]
1998
+ ):
1760
1999
  for d in destination:
1761
2000
  if not self.filecocoon(d):
1762
- raise OSError('Could not cocoon: {!s}'.format(d))
2001
+ raise OSError("Could not cocoon: {!s}".format(d))
1763
2002
  extras = list()
1764
2003
  hostname = self.fix_fthostname(hostname, fatal=False)
1765
2004
  logname = self.fix_ftuser(hostname, logname, fatal=False)
1766
2005
  if hostname:
1767
2006
  if port is not None:
1768
- hostname += ':{:s}'.format(port)
1769
- extras.extend(['-h', hostname])
2007
+ hostname += ":{:s}".format(port)
2008
+ extras.extend(["-h", hostname])
1770
2009
  if logname:
1771
- extras.extend(['-u', logname])
1772
- ftcmd = self.ftgetcmd or 'ftget'
1773
- plocale = locale.getlocale()[1] or 'ascii'
1774
- with tempfile.TemporaryFile(dir=self.path.dirname(self.path.abspath(destination[0])),
1775
- mode='wb') as tmpio:
1776
- tmpio.writelines(['{:s} {:s}\n'.format(s, d).encode(plocale)
1777
- for s, d in zip(source, destination)])
2010
+ extras.extend(["-u", logname])
2011
+ ftcmd = self.ftgetcmd or "ftget"
2012
+ plocale = locale.getlocale()[1] or "ascii"
2013
+ with tempfile.TemporaryFile(
2014
+ dir=self.path.dirname(self.path.abspath(destination[0])),
2015
+ mode="wb",
2016
+ ) as tmpio:
2017
+ tmpio.writelines(
2018
+ [
2019
+ "{:s} {:s}\n".format(s, d).encode(plocale)
2020
+ for s, d in zip(source, destination)
2021
+ ]
2022
+ )
1778
2023
  tmpio.seek(0)
1779
- with tempfile.TemporaryFile(dir=self.path.dirname(self.path.abspath(destination[0])),
1780
- mode='w+b') as tmpoutput:
2024
+ with tempfile.TemporaryFile(
2025
+ dir=self.path.dirname(self.path.abspath(destination[0])),
2026
+ mode="w+b",
2027
+ ) as tmpoutput:
1781
2028
  try:
1782
- rc = self.spawn([ftcmd, ] + extras, output=tmpoutput, stdin=tmpio)
2029
+ rc = self.spawn(
2030
+ [
2031
+ ftcmd,
2032
+ ]
2033
+ + extras,
2034
+ output=tmpoutput,
2035
+ stdin=tmpio,
2036
+ )
1783
2037
  except ExecutionError:
1784
2038
  rc = False
1785
2039
  # Process output data
1786
2040
  tmpoutput.seek(0)
1787
2041
  ft_outputs = tmpoutput.read()
1788
- ft_outputs = ft_outputs.decode(locale.getlocale()[1] or 'ascii', 'replace')
1789
- logger.info('Here is the ftget command output: \n%s', ft_outputs)
2042
+ ft_outputs = ft_outputs.decode(
2043
+ locale.getlocale()[1] or "ascii", "replace"
2044
+ )
2045
+ logger.info("Here is the ftget command output: \n%s", ft_outputs)
1790
2046
  # Expand the return codes
1791
2047
  if rc:
1792
- x_rc = [True, ] * len(source)
2048
+ x_rc = [
2049
+ True,
2050
+ ] * len(source)
1793
2051
  else:
1794
- ack_re = re.compile(r'.*FT_(OK|ABORT)\s*:\s*GET\s+(.*)$')
2052
+ ack_re = re.compile(r".*FT_(OK|ABORT)\s*:\s*GET\s+(.*)$")
1795
2053
  ack_lines = dict()
1796
- for line in ft_outputs.split('\n'):
2054
+ for line in ft_outputs.split("\n"):
1797
2055
  ack_match = ack_re.match(line)
1798
2056
  if ack_match:
1799
- ack_lines[ack_match.group(2)] = ack_match.group(1) == 'OK'
2057
+ ack_lines[ack_match.group(2)] = (
2058
+ ack_match.group(1) == "OK"
2059
+ )
1800
2060
  x_rc = []
1801
2061
  for a_source in source:
1802
2062
  my_rc = None
@@ -1806,16 +2066,28 @@ class OSExtended(System):
1806
2066
  break
1807
2067
  x_rc.append(my_rc)
1808
2068
  else:
1809
- raise OSError('Source or destination is not a plain file path: {!r}'.format(source))
2069
+ raise OSError(
2070
+ "Source or destination is not a plain file path: {!r}".format(
2071
+ source
2072
+ )
2073
+ )
1810
2074
  return x_rc
1811
2075
 
1812
2076
  def rawftput_worthy(self, source, destination):
1813
2077
  """Is it allowed to use FtServ given **source** and **destination**."""
1814
- return self.ftraw and self.ftserv_allowed(source, destination)
2078
+ return self.ftserv and self.ftserv_allowed(source, destination)
1815
2079
 
1816
2080
  @fmtshcmd
1817
- def rawftput(self, source, destination, hostname=None, logname=None, port=None,
1818
- cpipeline=None, sync=False):
2081
+ def rawftput(
2082
+ self,
2083
+ source,
2084
+ destination,
2085
+ hostname=None,
2086
+ logname=None,
2087
+ port=None,
2088
+ cpipeline=None,
2089
+ sync=False,
2090
+ ):
1819
2091
  """Proceed with some external ftput command on the specified target.
1820
2092
 
1821
2093
  :param str source: Path to the source filename
@@ -1829,21 +2101,43 @@ class OSExtended(System):
1829
2101
  """
1830
2102
  if cpipeline is not None:
1831
2103
  if cpipeline.compress2rawftp(source):
1832
- return self.ftserv_put(source, destination, hostname,
1833
- logname=logname, port=port,
1834
- specialshell=cpipeline.compress2rawftp(source),
1835
- sync=sync)
2104
+ return self.ftserv_put(
2105
+ source,
2106
+ destination,
2107
+ hostname,
2108
+ logname=logname,
2109
+ port=port,
2110
+ specialshell=cpipeline.compress2rawftp(source),
2111
+ sync=sync,
2112
+ )
1836
2113
  else:
1837
2114
  if port is None:
1838
2115
  port = DEFAULT_FTP_PORT
1839
- return self.ftput(source, destination, hostname, logname=logname,
1840
- port=port, cpipeline=cpipeline, sync=sync)
2116
+ return self.ftput(
2117
+ source,
2118
+ destination,
2119
+ hostname,
2120
+ logname=logname,
2121
+ port=port,
2122
+ cpipeline=cpipeline,
2123
+ sync=sync,
2124
+ )
1841
2125
  else:
1842
- return self.ftserv_put(source, destination, hostname, logname,
1843
- port=port, sync=sync)
2126
+ return self.ftserv_put(
2127
+ source, destination, hostname, logname, port=port, sync=sync
2128
+ )
1844
2129
 
1845
- def smartftput(self, source, destination, hostname=None, logname=None, port=None,
1846
- cpipeline=None, sync=False, fmt=None):
2130
+ def smartftput(
2131
+ self,
2132
+ source,
2133
+ destination,
2134
+ hostname=None,
2135
+ logname=None,
2136
+ port=None,
2137
+ cpipeline=None,
2138
+ sync=False,
2139
+ fmt=None,
2140
+ ):
1847
2141
  """Select the best alternative between ``ftput`` and ``rawftput``.
1848
2142
 
1849
2143
  :param source: The source of data (either a path to file or a
@@ -1862,26 +2156,53 @@ class OSExtended(System):
1862
2156
 
1863
2157
  ``rawftput`` will be used if all of the following conditions are met:
1864
2158
 
1865
- * ``self.ftraw`` is *True*
2159
+ * ``self.ftserv`` is *True*
1866
2160
  * **source** is a string (as opposed to a File like object)
1867
2161
  * **destination** is a string (as opposed to a File like object)
1868
2162
  """
1869
2163
  if self.rawftput_worthy(source, destination):
1870
- return self.rawftput(source, destination, hostname=hostname, logname=logname,
1871
- port=port, cpipeline=cpipeline, sync=sync, fmt=fmt)
2164
+ return self.rawftput(
2165
+ source,
2166
+ destination,
2167
+ hostname=hostname,
2168
+ logname=logname,
2169
+ port=port,
2170
+ cpipeline=cpipeline,
2171
+ sync=sync,
2172
+ fmt=fmt,
2173
+ )
1872
2174
  else:
1873
2175
  if port is None:
1874
2176
  port = DEFAULT_FTP_PORT
1875
- return self.ftput(source, destination, hostname=hostname, logname=logname,
1876
- port=port, cpipeline=cpipeline, sync=sync, fmt=fmt)
2177
+ return self.ftput(
2178
+ source,
2179
+ destination,
2180
+ hostname=hostname,
2181
+ logname=logname,
2182
+ port=port,
2183
+ cpipeline=cpipeline,
2184
+ sync=sync,
2185
+ fmt=fmt,
2186
+ )
1877
2187
 
1878
2188
  def rawftget_worthy(self, source, destination, cpipeline=None):
1879
2189
  """Is it allowed to use FtServ given **source** and **destination**."""
1880
- return self.ftraw and cpipeline is None and self.ftserv_allowed(source, destination)
2190
+ return (
2191
+ self.ftserv
2192
+ and cpipeline is None
2193
+ and self.ftserv_allowed(source, destination)
2194
+ )
1881
2195
 
1882
2196
  @fmtshcmd
1883
- def rawftget(self, source, destination, hostname=None, logname=None, port=None,
1884
- cpipeline=None):
2197
+ def rawftget(
2198
+ self,
2199
+ source,
2200
+ destination,
2201
+ hostname=None,
2202
+ logname=None,
2203
+ port=None,
2204
+ cpipeline=None,
2205
+ ):
1885
2206
  """Proceed with some external ftget command on the specified target.
1886
2207
 
1887
2208
  :param str source: the remote path to get data
@@ -1893,11 +2214,20 @@ class OSExtended(System):
1893
2214
  """
1894
2215
  if cpipeline is not None:
1895
2216
  raise OSError("cpipeline is not supported by this method.")
1896
- return self.ftserv_get(source, destination, hostname, logname, port=port)
2217
+ return self.ftserv_get(
2218
+ source, destination, hostname, logname, port=port
2219
+ )
1897
2220
 
1898
2221
  @fmtshcmd
1899
- def batchrawftget(self, source, destination, hostname=None, logname=None,
1900
- port=None, cpipeline=None):
2222
+ def batchrawftget(
2223
+ self,
2224
+ source,
2225
+ destination,
2226
+ hostname=None,
2227
+ logname=None,
2228
+ port=None,
2229
+ cpipeline=None,
2230
+ ):
1901
2231
  """Proceed with some external ftget command on the specified target.
1902
2232
 
1903
2233
  :param source: A list of remote paths to get data
@@ -1909,10 +2239,20 @@ class OSExtended(System):
1909
2239
  """
1910
2240
  if cpipeline is not None:
1911
2241
  raise OSError("cpipeline is not supported by this method.")
1912
- return self.ftserv_batchget(source, destination, hostname, logname, port=port)
2242
+ return self.ftserv_batchget(
2243
+ source, destination, hostname, logname, port=port
2244
+ )
1913
2245
 
1914
- def smartftget(self, source, destination, hostname=None, logname=None, port=None,
1915
- cpipeline=None, fmt=None):
2246
+ def smartftget(
2247
+ self,
2248
+ source,
2249
+ destination,
2250
+ hostname=None,
2251
+ logname=None,
2252
+ port=None,
2253
+ cpipeline=None,
2254
+ fmt=None,
2255
+ ):
1916
2256
  """Select the best alternative between ``ftget`` and ``rawftget``.
1917
2257
 
1918
2258
  :param str source: the remote path to get data
@@ -1930,23 +2270,45 @@ class OSExtended(System):
1930
2270
 
1931
2271
  ``rawftget`` will be used if all of the following conditions are met:
1932
2272
 
1933
- * ``self.ftraw`` is *True*
2273
+ * ``self.ftserv`` is *True*
1934
2274
  * **cpipeline** is None
1935
2275
  * **source** is a string (as opposed to a File like object)
1936
2276
  * **destination** is a string (as opposed to a File like object)
1937
2277
  """
1938
2278
  if self.rawftget_worthy(source, destination, cpipeline):
1939
2279
  # FtServ is uninteresting when dealing with compression
1940
- return self.rawftget(source, destination, hostname=hostname, logname=logname,
1941
- port=port, cpipeline=cpipeline, fmt=fmt)
2280
+ return self.rawftget(
2281
+ source,
2282
+ destination,
2283
+ hostname=hostname,
2284
+ logname=logname,
2285
+ port=port,
2286
+ cpipeline=cpipeline,
2287
+ fmt=fmt,
2288
+ )
1942
2289
  else:
1943
2290
  if port is None:
1944
2291
  port = DEFAULT_FTP_PORT
1945
- return self.ftget(source, destination, hostname=hostname, logname=logname,
1946
- port=port, cpipeline=cpipeline, fmt=fmt)
2292
+ return self.ftget(
2293
+ source,
2294
+ destination,
2295
+ hostname=hostname,
2296
+ logname=logname,
2297
+ port=port,
2298
+ cpipeline=cpipeline,
2299
+ fmt=fmt,
2300
+ )
1947
2301
 
1948
- def smartbatchftget(self, source, destination, hostname=None, logname=None,
1949
- port=None, cpipeline=None, fmt=None):
2302
+ def smartbatchftget(
2303
+ self,
2304
+ source,
2305
+ destination,
2306
+ hostname=None,
2307
+ logname=None,
2308
+ port=None,
2309
+ cpipeline=None,
2310
+ fmt=None,
2311
+ ):
1950
2312
  """
1951
2313
  Select the best alternative between ``ftget`` and ``batchrawftget``
1952
2314
  when retrieving several files.
@@ -1964,18 +2326,37 @@ class OSExtended(System):
1964
2326
  uncompress the data during the file transfer.
1965
2327
  :param str fmt: The format of data.
1966
2328
  """
1967
- if all([self.rawftget_worthy(s, d, cpipeline) for s, d in zip(source, destination)]):
2329
+ if all(
2330
+ [
2331
+ self.rawftget_worthy(s, d, cpipeline)
2332
+ for s, d in zip(source, destination)
2333
+ ]
2334
+ ):
1968
2335
  # FtServ is uninteresting when dealing with compression
1969
- return self.batchrawftget(source, destination, hostname=hostname, logname=logname,
1970
- port=None, cpipeline=cpipeline, fmt=fmt)
2336
+ return self.batchrawftget(
2337
+ source,
2338
+ destination,
2339
+ hostname=hostname,
2340
+ logname=logname,
2341
+ port=None,
2342
+ cpipeline=cpipeline,
2343
+ fmt=fmt,
2344
+ )
1971
2345
  else:
1972
2346
  rc = True
1973
2347
  if port is None:
1974
2348
  port = DEFAULT_FTP_PORT
1975
2349
  with self.ftppool():
1976
2350
  for s, d in zip(source, destination):
1977
- rc = rc and self.ftget(s, d, hostname=hostname, logname=logname,
1978
- port=port, cpipeline=cpipeline, fmt=fmt)
2351
+ rc = rc and self.ftget(
2352
+ s,
2353
+ d,
2354
+ hostname=hostname,
2355
+ logname=logname,
2356
+ port=port,
2357
+ cpipeline=cpipeline,
2358
+ fmt=fmt,
2359
+ )
1979
2360
  return rc
1980
2361
 
1981
2362
  def ssh(self, hostname, logname=None, *args, **kw):
@@ -1993,7 +2374,9 @@ class OSExtended(System):
1993
2374
  return AssistedSsh(self, hostname, logname, *args, **kw)
1994
2375
 
1995
2376
  @fmtshcmd
1996
- def scpput(self, source, destination, hostname, logname=None, cpipeline=None):
2377
+ def scpput(
2378
+ self, source, destination, hostname, logname=None, cpipeline=None
2379
+ ):
1997
2380
  """Perform an scp to the specified target.
1998
2381
 
1999
2382
  :param source: The source of data (either a path to file or a
@@ -2005,14 +2388,16 @@ class OSExtended(System):
2005
2388
  :param CompressionPipeline cpipeline: If not *None*, the object used to
2006
2389
  compress the data during the file transfer (default: *None*).
2007
2390
  """
2008
- logname = self.fix_ftuser(hostname, logname, fatal=False, defaults_to_user=False)
2009
- msg = '[hostname={!s} logname={!s}]'.format(hostname, logname)
2391
+ logname = self.fix_ftuser(
2392
+ hostname, logname, fatal=False, defaults_to_user=False
2393
+ )
2394
+ msg = "[hostname={!s} logname={!s}]".format(hostname, logname)
2010
2395
  ssh = self.ssh(hostname, logname)
2011
2396
  if isinstance(source, str) and cpipeline is None:
2012
- self.stderr('scpput', source, destination, msg)
2397
+ self.stderr("scpput", source, destination, msg)
2013
2398
  return ssh.scpput(source, destination)
2014
2399
  else:
2015
- self.stderr('scpput_stream', source, destination, msg)
2400
+ self.stderr("scpput_stream", source, destination, msg)
2016
2401
  if cpipeline is None:
2017
2402
  return ssh.scpput_stream(source, destination)
2018
2403
  else:
@@ -2020,7 +2405,9 @@ class OSExtended(System):
2020
2405
  return ssh.scpput_stream(csource, destination)
2021
2406
 
2022
2407
  @fmtshcmd
2023
- def scpget(self, source, destination, hostname, logname=None, cpipeline=None):
2408
+ def scpget(
2409
+ self, source, destination, hostname, logname=None, cpipeline=None
2410
+ ):
2024
2411
  """Perform an scp to get the specified source.
2025
2412
 
2026
2413
  :param str source: the remote path to get data
@@ -2032,14 +2419,16 @@ class OSExtended(System):
2032
2419
  :param CompressionPipeline cpipeline: If not *None*, the object used to
2033
2420
  uncompress the data during the file transfer (default: *None*).
2034
2421
  """
2035
- logname = self.fix_ftuser(hostname, logname, fatal=False, defaults_to_user=False)
2036
- msg = '[hostname={!s} logname={!s}]'.format(hostname, logname)
2422
+ logname = self.fix_ftuser(
2423
+ hostname, logname, fatal=False, defaults_to_user=False
2424
+ )
2425
+ msg = "[hostname={!s} logname={!s}]".format(hostname, logname)
2037
2426
  ssh = self.ssh(hostname, logname)
2038
2427
  if isinstance(destination, str) and cpipeline is None:
2039
- self.stderr('scpget', source, destination, msg)
2428
+ self.stderr("scpget", source, destination, msg)
2040
2429
  return ssh.scpget(source, destination)
2041
2430
  else:
2042
- self.stderr('scpget_stream', source, destination, msg)
2431
+ self.stderr("scpget_stream", source, destination, msg)
2043
2432
  if cpipeline is None:
2044
2433
  return ssh.scpget_stream(source, destination)
2045
2434
  else:
@@ -2048,7 +2437,7 @@ class OSExtended(System):
2048
2437
 
2049
2438
  def softlink(self, source, destination):
2050
2439
  """Set a symbolic link if **source** is not **destination**."""
2051
- self.stderr('softlink', source, destination)
2440
+ self.stderr("softlink", source, destination)
2052
2441
  if source == destination:
2053
2442
  return False
2054
2443
  else:
@@ -2057,7 +2446,7 @@ class OSExtended(System):
2057
2446
  def size(self, filepath):
2058
2447
  """Returns the actual size in bytes of the specified **filepath**."""
2059
2448
  filepath = self.path.expanduser(filepath)
2060
- self.stderr('size', filepath)
2449
+ self.stderr("size", filepath)
2061
2450
  try:
2062
2451
  return self.stat(filepath).st_size
2063
2452
  except Exception:
@@ -2076,7 +2465,9 @@ class OSExtended(System):
2076
2465
  total_size = self.size(objpath)
2077
2466
  for dirpath, dirnames, filenames in self.walk(objpath):
2078
2467
  for f in filenames + dirnames:
2079
- total_size += self.lstat(self.path.join(dirpath, f)).st_size
2468
+ total_size += self.lstat(
2469
+ self.path.join(dirpath, f)
2470
+ ).st_size
2080
2471
  return total_size
2081
2472
  return self.lstat(objpath).st_size
2082
2473
 
@@ -2084,8 +2475,8 @@ class OSExtended(System):
2084
2475
  """Normalises path name of **dirpath** and recursively creates this directory."""
2085
2476
  normdir = self.path.normpath(self.path.expanduser(dirpath))
2086
2477
  if normdir and not self.path.isdir(normdir):
2087
- logger.debug('Cocooning directory %s', normdir)
2088
- self.stderr('mkdir', normdir)
2478
+ logger.debug("Cocooning directory %s", normdir)
2479
+ self.stderr("mkdir", normdir)
2089
2480
  try:
2090
2481
  self.makedirs(normdir)
2091
2482
  return True
@@ -2102,17 +2493,17 @@ class OSExtended(System):
2102
2493
  """Normalises path name of ``destination`` and creates **destination**'s directory."""
2103
2494
  return self.mkdir(self.path.dirname(self.path.expanduser(destination)))
2104
2495
 
2105
- _SAFE_SUFFIX_RE = re.compile('_[a-f0-9]{32}$')
2496
+ _SAFE_SUFFIX_RE = re.compile("_[a-f0-9]{32}$")
2106
2497
 
2107
2498
  def safe_filesuffix(self):
2108
2499
  """Returns a file suffix that should be unique across the system."""
2109
- return '_' + uuid.uuid1().hex
2500
+ return "_" + uuid.uuid1().hex
2110
2501
 
2111
2502
  def safe_fileaddsuffix(self, name):
2112
2503
  """Returns a file path that will look like name + a unique suffix."""
2113
2504
  d_name = self.path.dirname(name)
2114
2505
  b_name = self.path.basename(name)
2115
- b_name = self._SAFE_SUFFIX_RE.sub('', b_name)
2506
+ b_name = self._SAFE_SUFFIX_RE.sub("", b_name)
2116
2507
  return self.path.join(d_name, b_name + self.safe_filesuffix())
2117
2508
 
2118
2509
  def _validate_symlink_below(self, symlink, valid_below):
@@ -2125,20 +2516,26 @@ class OSExtended(System):
2125
2516
  """
2126
2517
  link_to = self._os.readlink(symlink)
2127
2518
  # Is it relative ?
2128
- if re.match('^([^{0:s}]|..{0:s}|.{0:s})'.format(re.escape(os.path.sep)),
2129
- link_to):
2519
+ if re.match(
2520
+ "^([^{0:s}]|..{0:s}|.{0:s})".format(re.escape(os.path.sep)),
2521
+ link_to,
2522
+ ):
2130
2523
  symlink_dir = self.path.realpath(
2131
- self.path.abspath(
2132
- self.path.dirname(symlink)
2133
- )
2524
+ self.path.abspath(self.path.dirname(symlink))
2134
2525
  )
2135
2526
  abspath_to = self.path.normpath(
2136
2527
  self.path.join(symlink_dir, link_to)
2137
2528
  )
2138
2529
  # Valid ?
2139
- valid = self.path.commonprefix([valid_below, abspath_to]) == valid_below
2140
- return (self.path.relpath(abspath_to, start=symlink_dir)
2141
- if valid else None)
2530
+ valid = (
2531
+ self.path.commonprefix([valid_below, abspath_to])
2532
+ == valid_below
2533
+ )
2534
+ return (
2535
+ self.path.relpath(abspath_to, start=symlink_dir)
2536
+ if valid
2537
+ else None
2538
+ )
2142
2539
  else:
2143
2540
  return None
2144
2541
 
@@ -2150,10 +2547,11 @@ class OSExtended(System):
2150
2547
 
2151
2548
  The destination directory must not already exist.
2152
2549
  """
2153
- self.stderr('_copydatatree', src, dst)
2550
+ self.stderr("_copydatatree", src, dst)
2154
2551
  with self.mute_stderr():
2155
- keep_symlinks_below = (keep_symlinks_below or
2156
- self.path.realpath(self.path.abspath(src)))
2552
+ keep_symlinks_below = keep_symlinks_below or self.path.realpath(
2553
+ self.path.abspath(src)
2554
+ )
2157
2555
  names = self._os.listdir(src)
2158
2556
  self._os.makedirs(dst)
2159
2557
  errors = []
@@ -2162,14 +2560,19 @@ class OSExtended(System):
2162
2560
  dstname = self._os.path.join(dst, name)
2163
2561
  try:
2164
2562
  if self.path.isdir(srcname):
2165
- self._copydatatree(srcname, dstname,
2166
- keep_symlinks_below=keep_symlinks_below)
2563
+ self._copydatatree(
2564
+ srcname,
2565
+ dstname,
2566
+ keep_symlinks_below=keep_symlinks_below,
2567
+ )
2167
2568
  elif self._os.path.islink(srcname):
2168
- linkto = self._validate_symlink_below(srcname, keep_symlinks_below)
2569
+ linkto = self._validate_symlink_below(
2570
+ srcname, keep_symlinks_below
2571
+ )
2169
2572
  if linkto is not None:
2170
2573
  self._os.symlink(linkto, dstname)
2171
2574
  else:
2172
- rc = self._sh.copyfile(srcname, dstname)
2575
+ self._sh.copyfile(srcname, dstname)
2173
2576
  else:
2174
2577
  # Will raise a SpecialFileError for unsupported file types
2175
2578
  self._sh.copyfile(srcname, dstname)
@@ -2191,7 +2594,7 @@ class OSExtended(System):
2191
2594
  """
2192
2595
  source = self.path.expanduser(source)
2193
2596
  destination = self.path.expanduser(destination)
2194
- self.stderr('rawcp', source, destination)
2597
+ self.stderr("rawcp", source, destination)
2195
2598
  tmp = self.safe_fileaddsuffix(destination)
2196
2599
  if self.path.isdir(source):
2197
2600
  self._copydatatree(source, tmp)
@@ -2218,29 +2621,35 @@ class OSExtended(System):
2218
2621
  If **destination** is a real-word file name (i.e. not e File-like object),
2219
2622
  the operation is atomic.
2220
2623
  """
2221
- self.stderr('hybridcp', source, destination)
2624
+ self.stderr("hybridcp", source, destination)
2222
2625
  if isinstance(source, str):
2223
2626
  if not self.path.exists(source):
2224
2627
  if not silent:
2225
- logger.error('Missing source %s', source)
2628
+ logger.error("Missing source %s", source)
2226
2629
  return False
2227
- source = open(self.path.expanduser(source), 'rb')
2630
+ source = open(self.path.expanduser(source), "rb")
2228
2631
  xsource = True
2229
2632
  else:
2230
2633
  xsource = False
2231
2634
  try:
2232
2635
  source.seek(0)
2233
2636
  except AttributeError:
2234
- logger.warning('Could not rewind io source before cp: ' + str(source))
2637
+ logger.warning(
2638
+ "Could not rewind io source before cp: " + str(source)
2639
+ )
2235
2640
  if isinstance(destination, str):
2236
2641
  if self.filecocoon(destination):
2237
2642
  # Write to a temp file
2238
2643
  original_dest = self.path.expanduser(destination)
2239
- tmp_dest = self.safe_fileaddsuffix(self.path.expanduser(destination))
2240
- destination = open(tmp_dest, 'wb')
2644
+ tmp_dest = self.safe_fileaddsuffix(
2645
+ self.path.expanduser(destination)
2646
+ )
2647
+ destination = open(tmp_dest, "wb")
2241
2648
  xdestination = True
2242
2649
  else:
2243
- logger.error('Could not create a cocoon for file %s', destination)
2650
+ logger.error(
2651
+ "Could not create a cocoon for file %s", destination
2652
+ )
2244
2653
  return False
2245
2654
  else:
2246
2655
  destination.seek(0)
@@ -2253,8 +2662,13 @@ class OSExtended(System):
2253
2662
  if xdestination:
2254
2663
  destination.close()
2255
2664
  # Move the tmp_file to the real destination
2256
- if not self.move(tmp_dest, original_dest): # Move is atomic for a file
2257
- logger.error('Cannot move the tmp file to the final destination %s', original_dest)
2665
+ if not self.move(
2666
+ tmp_dest, original_dest
2667
+ ): # Move is atomic for a file
2668
+ logger.error(
2669
+ "Cannot move the tmp file to the final destination %s",
2670
+ original_dest,
2671
+ )
2258
2672
  return False
2259
2673
  return rc
2260
2674
 
@@ -2265,7 +2679,7 @@ class OSExtended(System):
2265
2679
  return st1.st_dev == st2.st_dev and not self.path.islink(path1)
2266
2680
 
2267
2681
  def _rawcp_instead_of_hardlink(self, source, destination, securecopy=True):
2268
- self.stderr('rawcp_instead_of_hardlink', source, destination)
2682
+ self.stderr("rawcp_instead_of_hardlink", source, destination)
2269
2683
  if securecopy:
2270
2684
  rc = self.rawcp(source, destination)
2271
2685
  else:
@@ -2294,19 +2708,29 @@ class OSExtended(System):
2294
2708
  except OSError as e:
2295
2709
  if e.errno == errno.EMLINK:
2296
2710
  # Too many links
2297
- logger.warning('Too many links for the source file (%s).', source)
2711
+ logger.warning(
2712
+ "Too many links for the source file (%s).", source
2713
+ )
2298
2714
  if self.usr_file(source):
2299
- rc = self._rawcp_instead_of_hardlink(source, destination, securecopy=securecopy)
2715
+ rc = self._rawcp_instead_of_hardlink(
2716
+ source, destination, securecopy=securecopy
2717
+ )
2300
2718
  if rc:
2301
2719
  try:
2302
- logger.warning('Replacing the orignal file with a copy...')
2720
+ logger.warning(
2721
+ "Replacing the orignal file with a copy..."
2722
+ )
2303
2723
  self.move(destination, source)
2304
2724
  except OSError as ebis:
2305
2725
  if ebis.errno == errno.EACCES:
2306
2726
  # Permission denied
2307
- logger.warning('No permissions to create a copy of the source file (%s)',
2308
- source)
2309
- logger.warning('Going on with the copy instead of the link...')
2727
+ logger.warning(
2728
+ "No permissions to create a copy of the source file (%s)",
2729
+ source,
2730
+ )
2731
+ logger.warning(
2732
+ "Going on with the copy instead of the link..."
2733
+ )
2310
2734
  else:
2311
2735
  raise
2312
2736
  else:
@@ -2319,9 +2743,15 @@ class OSExtended(System):
2319
2743
  rc = self.path.samefile(source, destination)
2320
2744
  return rc
2321
2745
 
2322
- def hardlink(self, source, destination,
2323
- link_threshold=0, readonly=True, securecopy=True,
2324
- keep_symlinks_below=None):
2746
+ def hardlink(
2747
+ self,
2748
+ source,
2749
+ destination,
2750
+ link_threshold=0,
2751
+ readonly=True,
2752
+ securecopy=True,
2753
+ keep_symlinks_below=None,
2754
+ ):
2325
2755
  """Create hardlinks for both single files or directories.
2326
2756
 
2327
2757
  :param int link_threshold: if the source file size is smaller than
@@ -2337,10 +2767,17 @@ class OSExtended(System):
2337
2767
  directory (if omitted, **source** is used)
2338
2768
  """
2339
2769
  if self.path.isdir(source):
2340
- self.stderr('hardlink', source, destination,
2341
- '#', 'directory,', 'readonly={!s}'.format(readonly))
2342
- keep_symlinks_below = (keep_symlinks_below or
2343
- self.path.realpath(self.path.abspath(source)))
2770
+ self.stderr(
2771
+ "hardlink",
2772
+ source,
2773
+ destination,
2774
+ "#",
2775
+ "directory,",
2776
+ "readonly={!s}".format(readonly),
2777
+ )
2778
+ keep_symlinks_below = keep_symlinks_below or self.path.realpath(
2779
+ self.path.abspath(source)
2780
+ )
2344
2781
  with self.mute_stderr():
2345
2782
  # Mimics 'cp -al'
2346
2783
  names = self._os.listdir(source)
@@ -2350,30 +2787,53 @@ class OSExtended(System):
2350
2787
  srcname = self._os.path.join(source, name)
2351
2788
  dstname = self._os.path.join(destination, name)
2352
2789
  if self._os.path.islink(srcname):
2353
- linkto = self._validate_symlink_below(srcname, keep_symlinks_below)
2790
+ linkto = self._validate_symlink_below(
2791
+ srcname, keep_symlinks_below
2792
+ )
2354
2793
  if linkto is None:
2355
- link_target = self.path.join(self.path.dirname(srcname),
2356
- self._os.readlink(srcname))
2357
- rc = self.hardlink(link_target, dstname,
2358
- link_threshold=link_threshold,
2359
- readonly=readonly, securecopy=securecopy,
2360
- keep_symlinks_below=keep_symlinks_below)
2794
+ link_target = self.path.join(
2795
+ self.path.dirname(srcname),
2796
+ self._os.readlink(srcname),
2797
+ )
2798
+ rc = self.hardlink(
2799
+ link_target,
2800
+ dstname,
2801
+ link_threshold=link_threshold,
2802
+ readonly=readonly,
2803
+ securecopy=securecopy,
2804
+ keep_symlinks_below=keep_symlinks_below,
2805
+ )
2361
2806
  else:
2362
2807
  self._os.symlink(linkto, dstname)
2363
2808
  elif self.path.isdir(srcname):
2364
- rc = self.hardlink(srcname, dstname,
2365
- link_threshold=link_threshold,
2366
- readonly=readonly, securecopy=securecopy,
2367
- keep_symlinks_below=keep_symlinks_below)
2809
+ rc = self.hardlink(
2810
+ srcname,
2811
+ dstname,
2812
+ link_threshold=link_threshold,
2813
+ readonly=readonly,
2814
+ securecopy=securecopy,
2815
+ keep_symlinks_below=keep_symlinks_below,
2816
+ )
2368
2817
  else:
2369
- if link_threshold and self.size(srcname) < link_threshold:
2370
- rc = self._rawcp_instead_of_hardlink(srcname, dstname, securecopy=securecopy)
2818
+ if (
2819
+ link_threshold
2820
+ and self.size(srcname) < link_threshold
2821
+ ):
2822
+ rc = self._rawcp_instead_of_hardlink(
2823
+ srcname, dstname, securecopy=securecopy
2824
+ )
2371
2825
  else:
2372
- rc = self._safe_hardlink(srcname, dstname, securecopy=securecopy)
2826
+ rc = self._safe_hardlink(
2827
+ srcname, dstname, securecopy=securecopy
2828
+ )
2373
2829
  if readonly and rc:
2374
2830
  self.readonly(dstname)
2375
2831
  if not rc:
2376
- logger.error('Error while processing %s (rc=%s)', srcname, str(rc))
2832
+ logger.error(
2833
+ "Error while processing %s (rc=%s)",
2834
+ srcname,
2835
+ str(rc),
2836
+ )
2377
2837
  break
2378
2838
  if rc:
2379
2839
  self._sh.copystat(source, destination)
@@ -2381,16 +2841,27 @@ class OSExtended(System):
2381
2841
  return rc
2382
2842
  else:
2383
2843
  if link_threshold and self.size(source) < link_threshold:
2384
- rc = self._rawcp_instead_of_hardlink(source, destination, securecopy=securecopy)
2844
+ rc = self._rawcp_instead_of_hardlink(
2845
+ source, destination, securecopy=securecopy
2846
+ )
2385
2847
  else:
2386
- self.stderr('hardlink', source, destination)
2387
- rc = self._safe_hardlink(source, destination, securecopy=securecopy)
2848
+ self.stderr("hardlink", source, destination)
2849
+ rc = self._safe_hardlink(
2850
+ source, destination, securecopy=securecopy
2851
+ )
2388
2852
  if readonly and rc:
2389
2853
  self.readonly(destination)
2390
2854
  return rc
2391
2855
 
2392
- def _smartcp_cross_users_links_fallback(self, source, destination, smartcp_threshold, silent,
2393
- exc, tmp_destination=None):
2856
+ def _smartcp_cross_users_links_fallback(
2857
+ self,
2858
+ source,
2859
+ destination,
2860
+ smartcp_threshold,
2861
+ silent,
2862
+ exc,
2863
+ tmp_destination=None,
2864
+ ):
2394
2865
  """Catch errors related to Kernel configuration."""
2395
2866
  if (exc.errno == errno.EPERM) and not self.usr_file(source):
2396
2867
  # This is expected to fail if the fs.protected_hardlinks
@@ -2400,8 +2871,12 @@ class OSExtended(System):
2400
2871
  logger.info("Force System's allow_cross_users_links to False")
2401
2872
  self.allow_cross_users_links = False
2402
2873
  logger.info("Re-running the smartcp command")
2403
- return self.smartcp(source, destination,
2404
- smartcp_threshold=smartcp_threshold, silent=silent)
2874
+ return self.smartcp(
2875
+ source,
2876
+ destination,
2877
+ smartcp_threshold=smartcp_threshold,
2878
+ silent=silent,
2879
+ )
2405
2880
  else:
2406
2881
  raise
2407
2882
 
@@ -2415,30 +2890,39 @@ class OSExtended(System):
2415
2890
  When working on a file, the operation is atomic. When working on a
2416
2891
  directory some restrictions apply (see :meth:`rawcp`)
2417
2892
  """
2418
- self.stderr('smartcp', source, destination)
2893
+ self.stderr("smartcp", source, destination)
2419
2894
  if not isinstance(source, str) or not isinstance(destination, str):
2420
2895
  return self.hybridcp(source, destination)
2421
2896
  source = self.path.expanduser(source)
2422
2897
  if not self.path.exists(source):
2423
2898
  if not silent:
2424
- logger.error('Missing source %s', source)
2899
+ logger.error("Missing source %s", source)
2425
2900
  return False
2426
2901
  if self.filecocoon(destination):
2427
2902
  destination = self.path.expanduser(destination)
2428
2903
  if self.path.islink(source):
2429
2904
  # Solve the symbolic link: this may avoid a rawcp
2430
2905
  source = self.path.realpath(source)
2431
- if (self.is_samefs(source, destination) and
2432
- (self.allow_cross_users_links or self.usr_file(source))):
2906
+ if self.is_samefs(source, destination) and (
2907
+ self.allow_cross_users_links or self.usr_file(source)
2908
+ ):
2433
2909
  tmp_destination = self.safe_fileaddsuffix(destination)
2434
2910
  if self.path.isdir(source):
2435
2911
  try:
2436
- rc = self.hardlink(source, tmp_destination,
2437
- link_threshold=smartcp_threshold, securecopy=False)
2912
+ rc = self.hardlink(
2913
+ source,
2914
+ tmp_destination,
2915
+ link_threshold=smartcp_threshold,
2916
+ securecopy=False,
2917
+ )
2438
2918
  except OSError as e:
2439
2919
  rc = self._smartcp_cross_users_links_fallback(
2440
- source, destination,
2441
- smartcp_threshold, silent, e, tmp_destination=tmp_destination
2920
+ source,
2921
+ destination,
2922
+ smartcp_threshold,
2923
+ silent,
2924
+ e,
2925
+ tmp_destination=tmp_destination,
2442
2926
  )
2443
2927
  else:
2444
2928
  if rc:
@@ -2446,44 +2930,71 @@ class OSExtended(System):
2446
2930
  with self.secure_directory_move(destination):
2447
2931
  rc = self.move(tmp_destination, destination)
2448
2932
  if not rc:
2449
- logger.error('Cannot move the tmp directory to the final destination %s',
2450
- destination)
2451
- self.remove(tmp_destination) # Anyway, try to clean-up things
2933
+ logger.error(
2934
+ "Cannot move the tmp directory to the final destination %s",
2935
+ destination,
2936
+ )
2937
+ self.remove(
2938
+ tmp_destination
2939
+ ) # Anyway, try to clean-up things
2452
2940
  else:
2453
- logger.error('Cannot copy the data to the tmp directory %s', tmp_destination)
2454
- self.remove(tmp_destination) # Anyway, try to clean-up things
2941
+ logger.error(
2942
+ "Cannot copy the data to the tmp directory %s",
2943
+ tmp_destination,
2944
+ )
2945
+ self.remove(
2946
+ tmp_destination
2947
+ ) # Anyway, try to clean-up things
2455
2948
  return rc
2456
2949
  else:
2457
2950
  try:
2458
- rc = self.hardlink(source, tmp_destination,
2459
- link_threshold=smartcp_threshold, securecopy=False)
2951
+ rc = self.hardlink(
2952
+ source,
2953
+ tmp_destination,
2954
+ link_threshold=smartcp_threshold,
2955
+ securecopy=False,
2956
+ )
2460
2957
  except OSError as e:
2461
- rc = self._smartcp_cross_users_links_fallback(source, destination,
2462
- smartcp_threshold, silent, e)
2958
+ rc = self._smartcp_cross_users_links_fallback(
2959
+ source, destination, smartcp_threshold, silent, e
2960
+ )
2463
2961
  else:
2464
- rc = rc and self.move(tmp_destination, destination) # Move is atomic for a file
2962
+ rc = rc and self.move(
2963
+ tmp_destination, destination
2964
+ ) # Move is atomic for a file
2465
2965
  # On some systems, the temporary file may remain (if the
2466
2966
  # destination's inode is identical to the tmp_destination's
2467
2967
  # inode). The following call to remove will remove leftovers.
2468
2968
  self.remove(tmp_destination)
2469
2969
  return rc
2470
2970
  else:
2471
- rc = self.rawcp(source, destination) # Rawcp is atomic as much as possible
2971
+ rc = self.rawcp(
2972
+ source, destination
2973
+ ) # Rawcp is atomic as much as possible
2472
2974
  if rc:
2473
2975
  if self.path.isdir(destination):
2474
2976
  for copiedfile in self.ffind(destination):
2475
- if not self.path.islink(copiedfile): # This make no sense to chmod symlinks
2977
+ if not self.path.islink(
2978
+ copiedfile
2979
+ ): # This make no sense to chmod symlinks
2476
2980
  self.chmod(copiedfile, 0o444)
2477
2981
  else:
2478
2982
  self.readonly(destination)
2479
2983
  return rc
2480
2984
  else:
2481
- logger.error('Could not create a cocoon for file %s', destination)
2985
+ logger.error("Could not create a cocoon for file %s", destination)
2482
2986
  return False
2483
2987
 
2484
2988
  @fmtshcmd
2485
- def cp(self, source, destination, intent='inout',
2486
- smartcp=True, smartcp_threshold=0, silent=False):
2989
+ def cp(
2990
+ self,
2991
+ source,
2992
+ destination,
2993
+ intent="inout",
2994
+ smartcp=True,
2995
+ smartcp_threshold=0,
2996
+ silent=False,
2997
+ ):
2487
2998
  """Copy the **source** file to a safe **destination**.
2488
2999
 
2489
3000
  :param source: The source of data (either a path to file or a
@@ -2504,27 +3015,31 @@ class OSExtended(System):
2504
3015
 
2505
3016
  The fastest option should be used...
2506
3017
  """
2507
- self.stderr('cp', source, destination)
3018
+ self.stderr("cp", source, destination)
2508
3019
  if not isinstance(source, str) or not isinstance(destination, str):
2509
3020
  return self.hybridcp(source, destination, silent=silent)
2510
3021
  if not self.path.exists(source):
2511
3022
  if not silent:
2512
- logger.error('Missing source %s', source)
3023
+ logger.error("Missing source %s", source)
2513
3024
  return False
2514
- if smartcp and intent == 'in':
2515
- return self.smartcp(source, destination,
2516
- smartcp_threshold=smartcp_threshold, silent=silent)
3025
+ if smartcp and intent == "in":
3026
+ return self.smartcp(
3027
+ source,
3028
+ destination,
3029
+ smartcp_threshold=smartcp_threshold,
3030
+ silent=silent,
3031
+ )
2517
3032
  if self.filecocoon(destination):
2518
3033
  return self.rawcp(source, destination)
2519
3034
  else:
2520
- logger.error('Could not create a cocoon for file %s', destination)
3035
+ logger.error("Could not create a cocoon for file %s", destination)
2521
3036
  return False
2522
3037
 
2523
3038
  def glob(self, *args):
2524
3039
  """Glob file system entries according to ``args``. Returns a list."""
2525
3040
  entries = []
2526
3041
  for entry in args:
2527
- if entry.startswith(':'):
3042
+ if entry.startswith(":"):
2528
3043
  entries.append(entry[1:])
2529
3044
  else:
2530
3045
  entries.extend(glob.glob(self.path.expanduser(entry)))
@@ -2544,15 +3059,23 @@ class OSExtended(System):
2544
3059
  """
2545
3060
  safe = True
2546
3061
  if len(thispath.split(self._os.sep)) < self._rmtreemin + 1:
2547
- logger.warning('Unsafe starting point depth %s (min is %s)', thispath, self._rmtreemin)
3062
+ logger.warning(
3063
+ "Unsafe starting point depth %s (min is %s)",
3064
+ thispath,
3065
+ self._rmtreemin,
3066
+ )
2548
3067
  safe = False
2549
3068
  else:
2550
3069
  for safepack in safedirs:
2551
3070
  (safedir, d) = safepack
2552
3071
  rp = self.path.relpath(thispath, safedir)
2553
- if not rp.startswith('..'):
3072
+ if not rp.startswith(".."):
2554
3073
  if len(rp.split(self._os.sep)) < d:
2555
- logger.warning('Unsafe access to %s relative to %s', thispath, safedir)
3074
+ logger.warning(
3075
+ "Unsafe access to %s relative to %s",
3076
+ thispath,
3077
+ safedir,
3078
+ )
2556
3079
  safe = False
2557
3080
  return safe
2558
3081
 
@@ -2565,43 +3088,47 @@ class OSExtended(System):
2565
3088
  if isinstance(pathlist, str):
2566
3089
  pathlist = [pathlist]
2567
3090
  for pname in pathlist:
2568
- for entry in filter(lambda x: self.safepath(x, safedirs), self.glob(pname)):
3091
+ for entry in filter(
3092
+ lambda x: self.safepath(x, safedirs), self.glob(pname)
3093
+ ):
2569
3094
  ok = self.remove(entry) and ok
2570
3095
  return ok
2571
3096
 
2572
3097
  def _globcmd(self, cmd, args, **kw):
2573
3098
  """Globbing files or directories as arguments before running ``cmd``."""
2574
- cmd.extend([opt for opt in args if opt.startswith('-')])
3099
+ cmd.extend([opt for opt in args if opt.startswith("-")])
2575
3100
  cmdlen = len(cmd)
2576
3101
  cmdargs = False
2577
- globtries = [self.path.expanduser(x) for x in args if not x.startswith('-')]
3102
+ globtries = [
3103
+ self.path.expanduser(x) for x in args if not x.startswith("-")
3104
+ ]
2578
3105
  for pname in globtries:
2579
3106
  cmdargs = True
2580
3107
  cmd.extend(self.glob(pname))
2581
3108
  if cmdargs and len(cmd) == cmdlen:
2582
- logger.warning('Could not find any matching pattern %s', globtries)
3109
+ logger.warning("Could not find any matching pattern %s", globtries)
2583
3110
  return False
2584
3111
  else:
2585
- kw.setdefault('ok', [0])
3112
+ kw.setdefault("ok", [0])
2586
3113
  return self.spawn(cmd, **kw)
2587
3114
 
2588
3115
  @_kw2spawn
2589
3116
  def wc(self, *args, **kw):
2590
3117
  """Word count on globbed files."""
2591
- return self._globcmd(['wc'], args, **kw)
3118
+ return self._globcmd(["wc"], args, **kw)
2592
3119
 
2593
3120
  @_kw2spawn
2594
3121
  def ls(self, *args, **kw):
2595
3122
  """Clone of the eponymous unix command."""
2596
- return self._globcmd(['ls'], args, **kw)
3123
+ return self._globcmd(["ls"], args, **kw)
2597
3124
 
2598
3125
  @_kw2spawn
2599
3126
  def ll(self, *args, **kw):
2600
3127
  """Clone of the eponymous unix alias (ls -l)."""
2601
- kw['output'] = True
2602
- llresult = self._globcmd(['ls', '-l'], args, **kw)
3128
+ kw["output"] = True
3129
+ llresult = self._globcmd(["ls", "-l"], args, **kw)
2603
3130
  if llresult:
2604
- for lline in [x for x in llresult if not x.startswith('total')]:
3131
+ for lline in [x for x in llresult if not x.startswith("total")]:
2605
3132
  print(lline)
2606
3133
  else:
2607
3134
  return False
@@ -2609,25 +3136,25 @@ class OSExtended(System):
2609
3136
  @_kw2spawn
2610
3137
  def dir(self, *args, **kw):
2611
3138
  """Proxy to ``ls('-l')``."""
2612
- return self._globcmd(['ls', '-l'], args, **kw)
3139
+ return self._globcmd(["ls", "-l"], args, **kw)
2613
3140
 
2614
3141
  @_kw2spawn
2615
3142
  def cat(self, *args, **kw):
2616
3143
  """Clone of the eponymous unix command."""
2617
- return self._globcmd(['cat'], args, **kw)
3144
+ return self._globcmd(["cat"], args, **kw)
2618
3145
 
2619
3146
  @fmtshcmd
2620
3147
  @_kw2spawn
2621
3148
  def diff(self, *args, **kw):
2622
3149
  """Clone of the eponymous unix command."""
2623
- kw.setdefault('ok', [0, 1])
2624
- kw.setdefault('output', False)
2625
- return self._globcmd(['cmp'], args, **kw)
3150
+ kw.setdefault("ok", [0, 1])
3151
+ kw.setdefault("output", False)
3152
+ return self._globcmd(["cmp"], args, **kw)
2626
3153
 
2627
3154
  @_kw2spawn
2628
3155
  def rmglob(self, *args, **kw):
2629
3156
  """Wrapper of the shell's ``rm`` command through the :meth:`globcmd` method."""
2630
- return self._globcmd(['rm'], args, **kw)
3157
+ return self._globcmd(["rm"], args, **kw)
2631
3158
 
2632
3159
  @fmtshcmd
2633
3160
  def move(self, source, destination):
@@ -2636,20 +3163,23 @@ class OSExtended(System):
2636
3163
  :param str source: The source object (file, directory, ...)
2637
3164
  :param str destination: The destination object (file, directory, ...)
2638
3165
  """
2639
- self.stderr('move', source, destination)
3166
+ self.stderr("move", source, destination)
2640
3167
  try:
2641
3168
  self._sh.move(source, destination)
2642
3169
  except Exception:
2643
- logger.critical('Could not move <%s> to <%s>', source, destination)
3170
+ logger.critical("Could not move <%s> to <%s>", source, destination)
2644
3171
  raise
2645
3172
  else:
2646
3173
  return True
2647
3174
 
2648
3175
  @contextlib.contextmanager
2649
3176
  def secure_directory_move(self, destination):
2650
- with self.lockdir_context(destination + '.vortex-lockdir', sloppy=True):
2651
- do_cleanup = (isinstance(destination, str) and
2652
- self.path.exists(destination))
3177
+ with self.lockdir_context(
3178
+ destination + ".vortex-lockdir", sloppy=True
3179
+ ):
3180
+ do_cleanup = isinstance(destination, str) and self.path.exists(
3181
+ destination
3182
+ )
2653
3183
  if do_cleanup:
2654
3184
  # Warning: Not an atomic portion of code (sorry)
2655
3185
  tmp_destination = self.safe_fileaddsuffix(destination)
@@ -2668,7 +3198,7 @@ class OSExtended(System):
2668
3198
  :param source: The source object (file, directory, File-like object, ...)
2669
3199
  :param destination: The destination object (file, directory, File-like object, ...)
2670
3200
  """
2671
- self.stderr('mv', source, destination)
3201
+ self.stderr("mv", source, destination)
2672
3202
  if not isinstance(source, str) or not isinstance(destination, str):
2673
3203
  self.hybridcp(source, destination)
2674
3204
  if isinstance(source, str):
@@ -2679,13 +3209,13 @@ class OSExtended(System):
2679
3209
  @_kw2spawn
2680
3210
  def mvglob(self, *args):
2681
3211
  """Wrapper of the shell's ``mv`` command through the :meth:`globcmd` method."""
2682
- return self._globcmd(['mv'], args)
3212
+ return self._globcmd(["mv"], args)
2683
3213
 
2684
3214
  def listdir(self, *args):
2685
3215
  """Proxy to standard :mod:`os` directory listing function."""
2686
3216
  if not args:
2687
- args = ('.',)
2688
- self.stderr('listdir', *args)
3217
+ args = (".",)
3218
+ self.stderr("listdir", *args)
2689
3219
  return self._os.listdir(self.path.expanduser(args[0]))
2690
3220
 
2691
3221
  def pyls(self, *args):
@@ -2694,10 +3224,10 @@ class OSExtended(System):
2694
3224
  :meth:`ls` method except that that shell's ``ls`` command is not actually
2695
3225
  called.
2696
3226
  """
2697
- rl = [x for x in args if not x.startswith('-')]
3227
+ rl = [x for x in args if not x.startswith("-")]
2698
3228
  if not rl:
2699
- rl.append('*')
2700
- self.stderr('pyls', *rl)
3229
+ rl.append("*")
3230
+ self.stderr("pyls", *rl)
2701
3231
  return self.glob(*rl)
2702
3232
 
2703
3233
  def ldirs(self, *args):
@@ -2706,23 +3236,23 @@ class OSExtended(System):
2706
3236
  :meth:`ls` method except that that shell's ``ls`` command is not actually
2707
3237
  called.
2708
3238
  """
2709
- rl = [x for x in args if not x.startswith('-')]
3239
+ rl = [x for x in args if not x.startswith("-")]
2710
3240
  if not rl:
2711
- rl.append('*')
2712
- self.stderr('ldirs', *rl)
3241
+ rl.append("*")
3242
+ self.stderr("ldirs", *rl)
2713
3243
  return [x for x in self.glob(*rl) if self.path.isdir(x)]
2714
3244
 
2715
3245
  @_kw2spawn
2716
3246
  def gzip(self, *args, **kw):
2717
3247
  """Simple gzip compression of a file."""
2718
- cmd = ['gzip', '-vf', args[0]]
3248
+ cmd = ["gzip", "-vf", args[0]]
2719
3249
  cmd.extend(args[1:])
2720
3250
  return self.spawn(cmd, **kw)
2721
3251
 
2722
3252
  @_kw2spawn
2723
3253
  def gunzip(self, *args, **kw):
2724
3254
  """Simple gunzip of a gzip-compressed file."""
2725
- cmd = ['gunzip', args[0]]
3255
+ cmd = ["gunzip", args[0]]
2726
3256
  cmd.extend(args[1:])
2727
3257
  return self.spawn(cmd, **kw)
2728
3258
 
@@ -2734,21 +3264,21 @@ class OSExtended(System):
2734
3264
  """Build a proper string sequence of tar options."""
2735
3265
  zopt = set(opts)
2736
3266
  if verbose:
2737
- zopt.add('v')
3267
+ zopt.add("v")
2738
3268
  else:
2739
- zopt.discard('v')
3269
+ zopt.discard("v")
2740
3270
  if autocompress:
2741
- if tarfile.endswith('gz'):
3271
+ if tarfile.endswith("gz"):
2742
3272
  # includes the conventional "*.tgz"
2743
- zopt.add('z')
3273
+ zopt.add("z")
2744
3274
  else:
2745
- zopt.discard('z')
2746
- if tarfile.endswith('bz') or tarfile.endswith('bz2'):
3275
+ zopt.discard("z")
3276
+ if tarfile.endswith("bz") or tarfile.endswith("bz2"):
2747
3277
  # includes the conventional "*.tbz"
2748
- zopt.add('j')
3278
+ zopt.add("j")
2749
3279
  else:
2750
- zopt.discard('j')
2751
- return ''.join(zopt)
3280
+ zopt.discard("j")
3281
+ return "".join(zopt)
2752
3282
 
2753
3283
  @_kw2spawn
2754
3284
  def tar(self, *args, **kw):
@@ -2756,8 +3286,13 @@ class OSExtended(System):
2756
3286
 
2757
3287
  :example: ``self.tar('destination.tar', 'directory1', 'directory2')``
2758
3288
  """
2759
- opts = self.taropts(args[0], 'cf', kw.pop('verbose', True), kw.pop('autocompress', True))
2760
- cmd = ['tar', opts, args[0]]
3289
+ opts = self.taropts(
3290
+ args[0],
3291
+ "cf",
3292
+ kw.pop("verbose", True),
3293
+ kw.pop("autocompress", True),
3294
+ )
3295
+ cmd = ["tar", opts, args[0]]
2761
3296
  cmd.extend(self.glob(*args[1:]))
2762
3297
  return self.spawn(cmd, **kw)
2763
3298
 
@@ -2768,8 +3303,13 @@ class OSExtended(System):
2768
3303
  :example: ``self.untar('source.tar')``
2769
3304
  :example: ``self.untar('source.tar', 'to_untar1', 'to_untar2')``
2770
3305
  """
2771
- opts = self.taropts(args[0], 'xf', kw.pop('verbose', True), kw.pop('autocompress', True))
2772
- cmd = ['tar', opts, args[0]]
3306
+ opts = self.taropts(
3307
+ args[0],
3308
+ "xf",
3309
+ kw.pop("verbose", True),
3310
+ kw.pop("autocompress", True),
3311
+ )
3312
+ cmd = ["tar", opts, args[0]]
2773
3313
  cmd.extend(args[1:])
2774
3314
  return self.spawn(cmd, **kw)
2775
3315
 
@@ -2784,56 +3324,69 @@ class OSExtended(System):
2784
3324
  This is done in a relatively safe way since it is checked that no existing
2785
3325
  files/directories are overwritten.
2786
3326
  """
2787
- uniquelevel_ignore = kw.pop('uniquelevel_ignore', False)
3327
+ uniquelevel_ignore = kw.pop("uniquelevel_ignore", False)
2788
3328
  fullsource = self.path.realpath(source)
2789
3329
  self.mkdir(destination)
2790
- loctmp = tempfile.mkdtemp(prefix='untar_', dir=destination)
3330
+ loctmp = tempfile.mkdtemp(prefix="untar_", dir=destination)
2791
3331
  with self.cdcontext(loctmp, clean_onexit=True):
2792
- output_setting = kw.pop('output', True)
3332
+ output_setting = kw.pop("output", True)
2793
3333
  output_txt = self.untar(fullsource, output=output_setting, **kw)
2794
3334
  if output_setting and output_txt:
2795
- logger.info('Untar command output:\n%s', '\n'.join(output_txt))
2796
- unpacked = self.glob('*')
2797
- unpacked_prefix = '.'
3335
+ logger.info("Untar command output:\n%s", "\n".join(output_txt))
3336
+ unpacked = self.glob("*")
3337
+ unpacked_prefix = "."
2798
3338
  # If requested, ignore the first level of directory
2799
- if (uniquelevel_ignore and len(unpacked) == 1 and
2800
- self.path.isdir(self.path.join(unpacked[0]))):
3339
+ if (
3340
+ uniquelevel_ignore
3341
+ and len(unpacked) == 1
3342
+ and self.path.isdir(self.path.join(unpacked[0]))
3343
+ ):
2801
3344
  unpacked_prefix = unpacked[0]
2802
- logger.info('Moving contents one level up: %s', unpacked_prefix)
3345
+ logger.info(
3346
+ "Moving contents one level up: %s", unpacked_prefix
3347
+ )
2803
3348
  with self.cdcontext(unpacked_prefix):
2804
- unpacked = self.glob('*')
3349
+ unpacked = self.glob("*")
2805
3350
  for untaritem in unpacked:
2806
- itemtarget = self.path.join('..', self.path.basename(untaritem))
3351
+ itemtarget = self.path.join(
3352
+ "..", self.path.basename(untaritem)
3353
+ )
2807
3354
  if self.path.exists(itemtarget):
2808
- logger.error('Some previous item exists before untar [%s]', untaritem)
3355
+ logger.error(
3356
+ "Some previous item exists before untar [%s]",
3357
+ untaritem,
3358
+ )
2809
3359
  else:
2810
- self.mv(self.path.join(unpacked_prefix, untaritem),
2811
- itemtarget)
3360
+ self.mv(
3361
+ self.path.join(unpacked_prefix, untaritem), itemtarget
3362
+ )
2812
3363
  return unpacked
2813
3364
 
2814
3365
  def is_tarname(self, objname):
2815
3366
  """Check if a ``objname`` is a string with ``.tar`` suffix."""
2816
- return isinstance(objname, str) and (objname.endswith('.tar') or
2817
- objname.endswith('.tar.gz') or
2818
- objname.endswith('.tgz') or
2819
- objname.endswith('.tar.bz2') or
2820
- objname.endswith('.tbz'))
3367
+ return isinstance(objname, str) and (
3368
+ objname.endswith(".tar")
3369
+ or objname.endswith(".tar.gz")
3370
+ or objname.endswith(".tgz")
3371
+ or objname.endswith(".tar.bz2")
3372
+ or objname.endswith(".tbz")
3373
+ )
2821
3374
 
2822
3375
  def tarname_radix(self, objname):
2823
3376
  """Remove any ``.tar`` specific suffix."""
2824
3377
  if not self.is_tarname(objname):
2825
3378
  return objname
2826
3379
  radix = self.path.splitext(objname)[0]
2827
- if radix.endswith('.tar'):
3380
+ if radix.endswith(".tar"):
2828
3381
  radix = radix[:-4]
2829
3382
  return radix
2830
3383
 
2831
3384
  def tarname_splitext(self, objname):
2832
3385
  """Like os.path.splitext, but for tar names (e.g. might return ``.tar.gz``)."""
2833
3386
  if not self.is_tarname(objname):
2834
- return (objname, '')
3387
+ return (objname, "")
2835
3388
  radix = self.tarname_radix(objname)
2836
- ext = objname.replace(radix, '')
3389
+ ext = objname.replace(radix, "")
2837
3390
  return (radix, ext)
2838
3391
 
2839
3392
  @fmtshcmd
@@ -2852,12 +3405,14 @@ class OSExtended(System):
2852
3405
  (either a file descriptor or a filename).
2853
3406
  """
2854
3407
  rc = None
2855
- if hasattr(destination, 'write'):
3408
+ if hasattr(destination, "write"):
2856
3409
  rc = gateway.dump(obj, destination, **opts)
2857
3410
  else:
2858
3411
  if self.filecocoon(destination):
2859
- with open(self.path.expanduser(destination),
2860
- 'w' + ('b' if bytesdump else '')) as fd:
3412
+ with open(
3413
+ self.path.expanduser(destination),
3414
+ "w" + ("b" if bytesdump else ""),
3415
+ ) as fd:
2861
3416
  rc = gateway.dump(obj, fd, **opts)
2862
3417
  return rc
2863
3418
 
@@ -2866,7 +3421,9 @@ class OSExtended(System):
2866
3421
  Dump a pickled representation of specified **obj** in file **destination**,
2867
3422
  (either a file descriptor or a filename).
2868
3423
  """
2869
- return self.blind_dump(pickle, obj, destination, bytesdump=True, **opts)
3424
+ return self.blind_dump(
3425
+ pickle, obj, destination, bytesdump=True, **opts
3426
+ )
2870
3427
 
2871
3428
  def json_dump(self, obj, destination, **opts):
2872
3429
  """
@@ -2880,11 +3437,12 @@ class OSExtended(System):
2880
3437
  Use **gateway** for a blind load the representation stored in file **source**,
2881
3438
  (either a file descriptor or a filename).
2882
3439
  """
2883
- if hasattr(source, 'read'):
3440
+ if hasattr(source, "read"):
2884
3441
  obj = gateway.load(source)
2885
3442
  else:
2886
- with open(self.path.expanduser(source),
2887
- 'r' + ('b' if bytesload else '')) as fd:
3443
+ with open(
3444
+ self.path.expanduser(source), "r" + ("b" if bytesload else "")
3445
+ ) as fd:
2888
3446
  obj = gateway.load(fd)
2889
3447
  return obj
2890
3448
 
@@ -2909,13 +3467,17 @@ class OSExtended(System):
2909
3467
  def utlines(self, *args):
2910
3468
  """Return number of significant code or configuration lines in specified directories."""
2911
3469
  lookfiles = [
2912
- x for x in self.ffind(*args)
2913
- if self.path.splitext[1] in ['.py', '.ini', '.tpl', '.rst']
3470
+ x
3471
+ for x in self.ffind(*args)
3472
+ if self.path.splitext[1] in [".py", ".ini", ".tpl", ".rst"]
2914
3473
  ]
2915
- return len([
2916
- x for x in self.cat(*lookfiles)
2917
- if re.search(r'\S', x) and re.search(r'[^\'\"\)\],\s]', x)
2918
- ])
3474
+ return len(
3475
+ [
3476
+ x
3477
+ for x in self.cat(*lookfiles)
3478
+ if re.search(r"\S", x) and re.search(r"[^\'\"\)\],\s]", x)
3479
+ ]
3480
+ )
2919
3481
 
2920
3482
  def _signal_intercept_init(self):
2921
3483
  """Initialise the signal handler object (but do not activate it)."""
@@ -2935,7 +3497,9 @@ class OSExtended(System):
2935
3497
  """
2936
3498
  self._sighandler.deactivate()
2937
3499
 
2938
- _LDD_REGEX = re.compile(r'^\s*([^\s]+)\s+=>\s*(?:([^\s]+)\s+\(0x.+\)|not found)$')
3500
+ _LDD_REGEX = re.compile(
3501
+ r"^\s*([^\s]+)\s+=>\s*(?:([^\s]+)\s+\(0x.+\)|not found)$"
3502
+ )
2939
3503
 
2940
3504
  def ldd(self, filename):
2941
3505
  """Call ldd on **filename**.
@@ -2943,14 +3507,14 @@ class OSExtended(System):
2943
3507
  Return the mapping between the library name and its physical path.
2944
3508
  """
2945
3509
  if self.path.isfile(filename):
2946
- ldd_out = self.spawn(('ldd', filename))
3510
+ ldd_out = self.spawn(("ldd", filename))
2947
3511
  libs = dict()
2948
3512
  for ldd_match in [self._LDD_REGEX.match(l) for l in ldd_out]:
2949
3513
  if ldd_match is not None:
2950
3514
  libs[ldd_match.group(1)] = ldd_match.group(2) or None
2951
3515
  return libs
2952
3516
  else:
2953
- raise ValueError('{} is not a regular file'.format(filename))
3517
+ raise ValueError("{} is not a regular file".format(filename))
2954
3518
 
2955
3519
  def generic_compress(self, pipelinedesc, source, destination=None):
2956
3520
  """Compress a file using the :class:`CompressionPipeline` class.
@@ -2964,7 +3528,9 @@ class OSExtended(System):
2964
3528
  if isinstance(source, str):
2965
3529
  destination = source + cp.suffix
2966
3530
  else:
2967
- raise ValueError("If destination is omitted, source must be a filename.")
3531
+ raise ValueError(
3532
+ "If destination is omitted, source must be a filename."
3533
+ )
2968
3534
  return cp.compress2file(source, destination)
2969
3535
 
2970
3536
  def generic_uncompress(self, pipelinedesc, source, destination=None):
@@ -2978,11 +3544,17 @@ class OSExtended(System):
2978
3544
  if destination is None:
2979
3545
  if isinstance(source, str):
2980
3546
  if source.endswith(cp.suffix):
2981
- destination = source[:-len(cp.suffix)]
3547
+ destination = source[: -len(cp.suffix)]
2982
3548
  else:
2983
- raise ValueError("Source do not exhibit the appropriate suffix ({:s})".format(cp.suffix))
3549
+ raise ValueError(
3550
+ "Source do not exhibit the appropriate suffix ({:s})".format(
3551
+ cp.suffix
3552
+ )
3553
+ )
2984
3554
  else:
2985
- raise ValueError("If destination is omitted, source must be a filename.")
3555
+ raise ValueError(
3556
+ "If destination is omitted, source must be a filename."
3557
+ )
2986
3558
  return cp.file2uncompress(source, destination)
2987
3559
 
2988
3560
  def find_mount_point(self, path):
@@ -2993,7 +3565,7 @@ class OSExtended(System):
2993
3565
  :rtype: str
2994
3566
  """
2995
3567
  if not self._os.path.exists(path):
2996
- logger.warning('Path does not exist: <%s>', path)
3568
+ logger.warning("Path does not exist: <%s>", path)
2997
3569
 
2998
3570
  path = self._os.path.abspath(path)
2999
3571
  while not self._os.path.ismount(path):
@@ -3013,7 +3585,9 @@ class OSExtended(System):
3013
3585
  """
3014
3586
  rc = None
3015
3587
  t0 = time.time()
3016
- while rc is None or (not rc and blocking and time.time() - t0 < timeout):
3588
+ while rc is None or (
3589
+ not rc and blocking and time.time() - t0 < timeout
3590
+ ):
3017
3591
  if rc is not None:
3018
3592
  self.sleep(sleeptime)
3019
3593
  try:
@@ -3021,7 +3595,7 @@ class OSExtended(System):
3021
3595
  # since we need to get an error if the target directory already
3022
3596
  # exists
3023
3597
  self._os.mkdir(ldir)
3024
- except FileExistsError as os_e:
3598
+ except FileExistsError:
3025
3599
  rc = False
3026
3600
  else:
3027
3601
  rc = True
@@ -3038,8 +3612,14 @@ class OSExtended(System):
3038
3612
  logger.warning("'%s' did not exists... that's odd", ldir)
3039
3613
 
3040
3614
  @contextlib.contextmanager
3041
- def lockdir_context(self, ldir,
3042
- sloppy=False, timeout=120, sleeptime_min=0.1, sleeptime_max=0.3):
3615
+ def lockdir_context(
3616
+ self,
3617
+ ldir,
3618
+ sloppy=False,
3619
+ timeout=120,
3620
+ sleeptime_min=0.1,
3621
+ sleeptime_max=0.3,
3622
+ ):
3043
3623
  """Try to acquire a lock directory and after that remove it.
3044
3624
 
3045
3625
  :param bool sloppy: If the lock can be acquired after *timeout* second, go on anyway
@@ -3049,14 +3629,20 @@ class OSExtended(System):
3049
3629
  :param float sleeptime_max: When blocking, wait at most **sleeptime_max** seconds
3050
3630
  between to attempts to acquire the lock.
3051
3631
  """
3052
- sleeptime = sleeptime_min + (sleeptime_max - sleeptime_min) * random.random()
3632
+ sleeptime = (
3633
+ sleeptime_min + (sleeptime_max - sleeptime_min) * random.random()
3634
+ )
3053
3635
  self.filecocoon(ldir)
3054
- rc = self._lockdir_create(ldir, blocking=True, timeout=timeout, sleeptime=sleeptime)
3636
+ rc = self._lockdir_create(
3637
+ ldir, blocking=True, timeout=timeout, sleeptime=sleeptime
3638
+ )
3055
3639
  try:
3056
3640
  if not rc:
3057
- msg = "Could not acquire lockdir < {:s} >. Already exists.".format(ldir)
3641
+ msg = "Could not acquire lockdir < {:s} >. Already exists.".format(
3642
+ ldir
3643
+ )
3058
3644
  if sloppy:
3059
- logger.warning(msg + '.. but going on.')
3645
+ logger.warning(msg + ".. but going on.")
3060
3646
  else:
3061
3647
  raise OSError(msg)
3062
3648
  yield
@@ -3070,9 +3656,11 @@ class OSExtended(System):
3070
3656
  if self.glove is not None:
3071
3657
  myglove = self.glove
3072
3658
  rcdir = myglove.configrc
3073
- lockdir = self.path.join(rcdir,
3074
- 'appwide_locks',
3075
- '{0.vapp:s}-{0.vconf:s}'.format(myglove))
3659
+ lockdir = self.path.join(
3660
+ rcdir,
3661
+ "appwide_locks",
3662
+ "{0.vapp:s}-{0.vconf:s}".format(myglove),
3663
+ )
3076
3664
  self.mkdir(lockdir)
3077
3665
  return lockdir
3078
3666
  else:
@@ -3097,10 +3685,9 @@ class OSExtended(System):
3097
3685
  attempts to acquire the lock.
3098
3686
  """
3099
3687
  ldir = self._appwide_lockdir_path(label)
3100
- return self._lockdir_create(ldir,
3101
- blocking=blocking,
3102
- timeout=timeout,
3103
- sleeptime=sleeptime)
3688
+ return self._lockdir_create(
3689
+ ldir, blocking=blocking, timeout=timeout, sleeptime=sleeptime
3690
+ )
3104
3691
 
3105
3692
  def appwide_unlock(self, label):
3106
3693
  """Pseudo-lock mechanism based on atomic directory creation: release lock.
@@ -3124,9 +3711,8 @@ class Python34:
3124
3711
  """
3125
3712
 
3126
3713
  # Optional, netcdf comparison tool
3127
- b_netcdf_checker = ExternalCodeImportChecker('netdcf')
3128
- with b_netcdf_checker as npregister:
3129
- from bronx.datagrip import netcdf as b_netcdf
3714
+ b_netcdf_checker = ExternalCodeImportChecker("netdcf")
3715
+ from bronx.datagrip import netcdf as b_netcdf
3130
3716
 
3131
3717
  if b_netcdf_checker.is_available():
3132
3718
  # Unfortunately, the netCDF4 package seems to leak memory,
@@ -3136,25 +3722,28 @@ class Python34:
3136
3722
  """Function started by the subprocess."""
3137
3723
  outcome.value = int(b_netcdf.netcdf_file_diff(nc1, nc2))
3138
3724
 
3139
- rc = multiprocessing.Value('i', 0)
3140
- p = multiprocessing.Process(target=_compare_function,
3141
- args=(netcdf1, netcdf2, rc))
3725
+ rc = multiprocessing.Value("i", 0)
3726
+ p = multiprocessing.Process(
3727
+ target=_compare_function, args=(netcdf1, netcdf2, rc)
3728
+ )
3142
3729
  p.start()
3143
3730
  p.join()
3144
3731
  return bool(rc.value)
3145
3732
  else:
3146
- logger.error("Unable to load the 'bronx.datagrip.netcdf' package. " +
3147
- "The netcdf library and/or 'netCDF4' python package are probably missing.")
3733
+ logger.error(
3734
+ "Unable to load the 'bronx.datagrip.netcdf' package. "
3735
+ + "The netcdf library and/or 'netCDF4' python package are probably missing."
3736
+ )
3148
3737
  return False
3149
3738
 
3150
3739
  # Let's make this method compatible with fmtshcmd...
3151
3740
  netcdf_diff.func_extern = True
3152
3741
 
3153
3742
 
3154
- _python34_fp = footprints.Footprint(info='An abstract footprint to be used with the Python34 Mixin',
3155
- only=dict(
3156
- after_python=PythonSimplifiedVersion('3.4.0')
3157
- ))
3743
+ _python34_fp = footprints.Footprint(
3744
+ info="An abstract footprint to be used with the Python34 Mixin",
3745
+ only=dict(after_python=PythonSimplifiedVersion("3.4.0")),
3746
+ )
3158
3747
 
3159
3748
 
3160
3749
  class Garbage(OSExtended):
@@ -3166,20 +3755,18 @@ class Garbage(OSExtended):
3166
3755
 
3167
3756
  _abstract = True
3168
3757
  _footprint = dict(
3169
- info = 'Garbage base system',
3170
- attr = dict(
3171
- sysname = dict(
3172
- outcast = ['Linux', 'Darwin', 'UnitTestLinux', 'UnitTestable']
3758
+ info="Garbage base system",
3759
+ attr=dict(
3760
+ sysname=dict(
3761
+ outcast=["Linux", "Darwin", "UnitTestLinux", "UnitTestable"]
3173
3762
  )
3174
3763
  ),
3175
- priority = dict(
3176
- level = footprints.priorities.top.DEFAULT
3177
- )
3764
+ priority=dict(level=footprints.priorities.top.DEFAULT),
3178
3765
  )
3179
3766
 
3180
3767
  def __init__(self, *args, **kw):
3181
3768
  """Gateway to parent method after debug logging."""
3182
- logger.debug('Garbage system init %s', self.__class__)
3769
+ logger.debug("Garbage system init %s", self.__class__)
3183
3770
  super().__init__(*args, **kw)
3184
3771
 
3185
3772
 
@@ -3188,7 +3775,7 @@ class Garbage34p(Garbage, Python34):
3188
3775
 
3189
3776
  _footprint = [
3190
3777
  _python34_fp,
3191
- dict(info = 'Garbage base system withh a blazing Python version')
3778
+ dict(info="Garbage base system withh a blazing Python version"),
3192
3779
  ]
3193
3780
 
3194
3781
 
@@ -3197,12 +3784,8 @@ class Linux(OSExtended):
3197
3784
 
3198
3785
  _abstract = True
3199
3786
  _footprint = dict(
3200
- info = 'Abstract Linux base system',
3201
- attr = dict(
3202
- sysname = dict(
3203
- values = ['Linux']
3204
- )
3205
- )
3787
+ info="Abstract Linux base system",
3788
+ attr=dict(sysname=dict(values=["Linux"])),
3206
3789
  )
3207
3790
 
3208
3791
  def __init__(self, *args, **kw):
@@ -3212,78 +3795,86 @@ class Linux(OSExtended):
3212
3795
 
3213
3796
  * **psopts** - as default option for the ps command (default: ``-w -f -a``).
3214
3797
  """
3215
- logger.debug('Linux system init %s', self.__class__)
3216
- self._psopts = kw.pop('psopts', ['-w', '-f', '-a'])
3798
+ logger.debug("Linux system init %s", self.__class__)
3799
+ self._psopts = kw.pop("psopts", ["-w", "-f", "-a"])
3217
3800
  super().__init__(*args, **kw)
3218
- self.__dict__['_cpusinfo'] = LinuxCpusInfo()
3801
+ self.__dict__["_cpusinfo"] = LinuxCpusInfo()
3219
3802
  try:
3220
- self.__dict__['_numainfo'] = LibNumaNodesInfo()
3803
+ self.__dict__["_numainfo"] = LibNumaNodesInfo()
3221
3804
  except (OSError, NotImplementedError):
3222
3805
  # On very few Linux systems, libnuma is not available...
3223
3806
  pass
3224
- self.__dict__['_memoryinfo'] = LinuxMemInfo()
3225
- self.__dict__['_netstatsinfo'] = LinuxNetstats()
3807
+ self.__dict__["_memoryinfo"] = LinuxMemInfo()
3808
+ self.__dict__["_netstatsinfo"] = LinuxNetstats()
3226
3809
 
3227
3810
  @property
3228
3811
  def realkind(self):
3229
- return 'linux'
3812
+ return "linux"
3230
3813
 
3231
- def cpus_ids_per_blocks(self, blocksize=1, topology='raw', hexmask=False):
3814
+ def cpus_ids_per_blocks(self, blocksize=1, topology="raw", hexmask=False):
3232
3815
  """Get the list of CPUs IDs for nicely ordered for subsequent binding.
3233
3816
 
3234
3817
  :param int blocksize: the number of thread consumed by one task
3235
3818
  :param str topology: The task distribution scheme
3236
3819
  :param bool hexmask: Return a list of CPU masks in hexadecimal
3237
3820
  """
3238
- if topology.startswith('numa'):
3239
- if topology.endswith('_discardsmt'):
3821
+ if topology.startswith("numa"):
3822
+ if topology.endswith("_discardsmt"):
3240
3823
  topology = topology[:-11]
3241
3824
  smtlayout = None
3242
3825
  else:
3243
3826
  smtlayout = self.cpus_info.physical_cores_smtthreads
3244
3827
  try:
3245
- cpulist = getattr(self.numa_info,
3246
- topology + '_cpulist')(blocksize,
3247
- smtlayout=smtlayout)
3828
+ cpulist = getattr(self.numa_info, topology + "_cpulist")(
3829
+ blocksize, smtlayout=smtlayout
3830
+ )
3248
3831
  except AttributeError:
3249
- raise ValueError('Unknown topology ({:s}).'.format(topology))
3832
+ raise ValueError("Unknown topology ({:s}).".format(topology))
3250
3833
  else:
3251
3834
  try:
3252
- cpulist = getattr(self.cpus_info, topology + '_cpulist')(blocksize)
3835
+ cpulist = getattr(self.cpus_info, topology + "_cpulist")(
3836
+ blocksize
3837
+ )
3253
3838
  except AttributeError:
3254
- raise ValueError('Unknown topology ({:s}).'.format(topology))
3839
+ raise ValueError("Unknown topology ({:s}).".format(topology))
3255
3840
  cpulist = list(cpulist)
3256
- cpulist = [[cpulist[(taskid * blocksize + i)]
3257
- for i in range(blocksize)]
3258
- for taskid in range(len(cpulist) // blocksize)]
3841
+ cpulist = [
3842
+ [cpulist[(taskid * blocksize + i)] for i in range(blocksize)]
3843
+ for taskid in range(len(cpulist) // blocksize)
3844
+ ]
3259
3845
  if hexmask:
3260
3846
  cpulist = [hex(sum([1 << i for i in item])) for item in cpulist]
3261
3847
  return cpulist
3262
3848
 
3263
- def cpus_ids_dispenser(self, topology='raw'):
3849
+ def cpus_ids_dispenser(self, topology="raw"):
3264
3850
  """Get a dispenser of CPUs IDs for nicely ordered for subsequent binding.
3265
3851
 
3266
3852
  :param str topology: The task distribution scheme
3267
3853
  """
3268
- if topology.startswith('numa'):
3269
- if topology.endswith('_discardsmt'):
3854
+ if topology.startswith("numa"):
3855
+ if topology.endswith("_discardsmt"):
3270
3856
  topology = topology[:-11]
3271
3857
  smtlayout = None
3272
3858
  else:
3273
3859
  smtlayout = self.cpus_info.physical_cores_smtthreads
3274
3860
  try:
3275
- cpudisp = getattr(self.numa_info,
3276
- topology + '_cpu_dispenser')(smtlayout=smtlayout)
3861
+ cpudisp = getattr(self.numa_info, topology + "_cpu_dispenser")(
3862
+ smtlayout=smtlayout
3863
+ )
3277
3864
  except AttributeError:
3278
- raise ValueError('Unknown topology ({:s}).'.format(topology))
3865
+ raise ValueError("Unknown topology ({:s}).".format(topology))
3279
3866
  else:
3280
3867
  try:
3281
- cpudisp = getattr(self.cpus_info, topology + '_cpu_dispenser')()
3868
+ cpudisp = getattr(
3869
+ self.cpus_info, topology + "_cpu_dispenser"
3870
+ )()
3282
3871
  except AttributeError:
3283
- raise ValueError('Unknown topology ({:s}).'.format(topology))
3872
+ raise ValueError("Unknown topology ({:s}).".format(topology))
3284
3873
  return cpudisp
3285
3874
 
3286
- def cpus_affinity_get(self, taskid, blocksize=1, topology='socketpacked', method='taskset'):
3875
+ def cpus_affinity_get(
3876
+ self, taskid, blocksize=1, topology="socketpacked", method="taskset"
3877
+ ):
3287
3878
  """Get the necessary command/environment to set the CPUs affinity.
3288
3879
 
3289
3880
  :param int taskid: the task number
@@ -3293,25 +3884,29 @@ class Linux(OSExtended):
3293
3884
  :return: A 3-elements tuple. (bool: BindingPossible,
3294
3885
  list: Starting command prefix, dict: Environment update)
3295
3886
  """
3296
- if method not in ('taskset', 'gomp', 'omp', 'ompverbose'):
3297
- raise ValueError('Unknown binding method ({:s}).'.format(method))
3298
- if method == 'taskset':
3299
- if not self.which('taskset'):
3300
- logger.warning("The taskset is program is missing. Going on without binding.")
3887
+ if method not in ("taskset", "gomp", "omp", "ompverbose"):
3888
+ raise ValueError("Unknown binding method ({:s}).".format(method))
3889
+ if method == "taskset":
3890
+ if not self.which("taskset"):
3891
+ logger.warning(
3892
+ "The taskset is program is missing. Going on without binding."
3893
+ )
3301
3894
  return (False, list(), dict())
3302
- cpulist = self.cpus_ids_per_blocks(blocksize=blocksize, topology=topology)
3895
+ cpulist = self.cpus_ids_per_blocks(
3896
+ blocksize=blocksize, topology=topology
3897
+ )
3303
3898
  cpus = cpulist[taskid % len(cpulist)]
3304
3899
  cmdl = list()
3305
3900
  env = dict()
3306
- if method == 'taskset':
3307
- cmdl += ['taskset', '--cpu-list', ','.join([str(c) for c in cpus])]
3308
- elif method == 'gomp':
3309
- env['GOMP_CPU_AFFINITY'] = ' '.join([str(c) for c in cpus])
3310
- elif method.startswith('omp'):
3311
- env['OMP_PLACES'] = ','.join(['{{{:d}}}'.format(c) for c in cpus])
3312
- if method.endswith('verbose'):
3313
- env['OMP_DISPLAY_ENV'] = 'TRUE'
3314
- env['OMP_DISPLAY_AFFINITY'] = 'TRUE'
3901
+ if method == "taskset":
3902
+ cmdl += ["taskset", "--cpu-list", ",".join([str(c) for c in cpus])]
3903
+ elif method == "gomp":
3904
+ env["GOMP_CPU_AFFINITY"] = " ".join([str(c) for c in cpus])
3905
+ elif method.startswith("omp"):
3906
+ env["OMP_PLACES"] = ",".join(["{{{:d}}}".format(c) for c in cpus])
3907
+ if method.endswith("verbose"):
3908
+ env["OMP_DISPLAY_ENV"] = "TRUE"
3909
+ env["OMP_DISPLAY_AFFINITY"] = "TRUE"
3315
3910
  return (True, cmdl, env)
3316
3911
 
3317
3912
 
@@ -3320,7 +3915,7 @@ class Linux34p(Linux, Python34):
3320
3915
 
3321
3916
  _footprint = [
3322
3917
  _python34_fp,
3323
- dict(info = 'Linux based system with a blazing Python version')
3918
+ dict(info="Linux based system with a blazing Python version"),
3324
3919
  ]
3325
3920
 
3326
3921
 
@@ -3328,26 +3923,24 @@ class LinuxDebug(Linux34p):
3328
3923
  """Special system class for crude debugging on Linux based systems."""
3329
3924
 
3330
3925
  _footprint = dict(
3331
- info = 'Linux debug system',
3332
- attr = dict(
3333
- version = dict(
3334
- optional = False,
3335
- values = ['dbug', 'debug'],
3336
- remap = dict(
3337
- dbug = 'debug'
3338
- )
3926
+ info="Linux debug system",
3927
+ attr=dict(
3928
+ version=dict(
3929
+ optional=False,
3930
+ values=["dbug", "debug"],
3931
+ remap=dict(dbug="debug"),
3339
3932
  )
3340
- )
3933
+ ),
3341
3934
  )
3342
3935
 
3343
3936
  def __init__(self, *args, **kw):
3344
3937
  """Gateway to parent method after debug logging."""
3345
- logger.debug('LinuxDebug system init %s', self.__class__)
3938
+ logger.debug("LinuxDebug system init %s", self.__class__)
3346
3939
  super().__init__(*args, **kw)
3347
3940
 
3348
3941
  @property
3349
3942
  def realkind(self):
3350
- return 'linuxdebug'
3943
+ return "linuxdebug"
3351
3944
 
3352
3945
 
3353
3946
  class Macosx(OSExtended):
@@ -3355,15 +3948,11 @@ class Macosx(OSExtended):
3355
3948
 
3356
3949
  _abstract = True
3357
3950
  _footprint = dict(
3358
- info = 'Apple Mac computer under Macosx',
3359
- attr = dict(
3360
- sysname = dict(
3361
- values = ['Darwin']
3362
- ),
3951
+ info="Apple Mac computer under Macosx",
3952
+ attr=dict(
3953
+ sysname=dict(values=["Darwin"]),
3363
3954
  ),
3364
- priority = dict(
3365
- level = footprints.priorities.top.TOOLBOX
3366
- )
3955
+ priority=dict(level=footprints.priorities.top.TOOLBOX),
3367
3956
  )
3368
3957
 
3369
3958
  def __init__(self, *args, **kw):
@@ -3373,18 +3962,18 @@ class Macosx(OSExtended):
3373
3962
 
3374
3963
  * **psopts** - as default option for the ps command (default: ``-w -f -a``).
3375
3964
  """
3376
- logger.debug('Darwin system init %s', self.__class__)
3377
- self._psopts = kw.pop('psopts', ['-w', '-f', '-a'])
3965
+ logger.debug("Darwin system init %s", self.__class__)
3966
+ self._psopts = kw.pop("psopts", ["-w", "-f", "-a"])
3378
3967
  super().__init__(*args, **kw)
3379
3968
 
3380
3969
  @property
3381
3970
  def realkind(self):
3382
- return 'darwin'
3971
+ return "darwin"
3383
3972
 
3384
3973
  @property
3385
3974
  def default_syslog(self):
3386
3975
  """Address to use in logging.handler.SysLogHandler()."""
3387
- return '/var/run/syslog'
3976
+ return "/var/run/syslog"
3388
3977
 
3389
3978
 
3390
3979
  class Macosx34p(Macosx, Python34):
@@ -3392,5 +3981,7 @@ class Macosx34p(Macosx, Python34):
3392
3981
 
3393
3982
  _footprint = [
3394
3983
  _python34_fp,
3395
- dict(info = 'Apple Mac computer under Macosx with a blazing Python version')
3984
+ dict(
3985
+ info="Apple Mac computer under Macosx with a blazing Python version"
3986
+ ),
3396
3987
  ]