vortex-nwp 2.0.0b1__py3-none-any.whl → 2.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- vortex/__init__.py +75 -47
- vortex/algo/__init__.py +3 -2
- vortex/algo/components.py +944 -618
- vortex/algo/mpitools.py +802 -497
- vortex/algo/mpitools_templates/__init__.py +1 -0
- vortex/algo/serversynctools.py +34 -33
- vortex/config.py +19 -22
- vortex/data/__init__.py +9 -3
- vortex/data/abstractstores.py +593 -655
- vortex/data/containers.py +217 -162
- vortex/data/contents.py +65 -39
- vortex/data/executables.py +93 -102
- vortex/data/flow.py +40 -34
- vortex/data/geometries.py +228 -132
- vortex/data/handlers.py +436 -227
- vortex/data/outflow.py +15 -15
- vortex/data/providers.py +185 -163
- vortex/data/resources.py +48 -42
- vortex/data/stores.py +540 -417
- vortex/data/sync_templates/__init__.py +0 -0
- vortex/gloves.py +114 -87
- vortex/layout/__init__.py +1 -8
- vortex/layout/contexts.py +150 -84
- vortex/layout/dataflow.py +353 -202
- vortex/layout/monitor.py +264 -128
- vortex/nwp/__init__.py +5 -2
- vortex/nwp/algo/__init__.py +14 -5
- vortex/nwp/algo/assim.py +205 -151
- vortex/nwp/algo/clim.py +683 -517
- vortex/nwp/algo/coupling.py +447 -225
- vortex/nwp/algo/eda.py +437 -229
- vortex/nwp/algo/eps.py +403 -231
- vortex/nwp/algo/forecasts.py +416 -275
- vortex/nwp/algo/fpserver.py +683 -307
- vortex/nwp/algo/ifsnaming.py +205 -145
- vortex/nwp/algo/ifsroot.py +215 -122
- vortex/nwp/algo/monitoring.py +137 -76
- vortex/nwp/algo/mpitools.py +330 -190
- vortex/nwp/algo/odbtools.py +637 -353
- vortex/nwp/algo/oopsroot.py +454 -273
- vortex/nwp/algo/oopstests.py +90 -56
- vortex/nwp/algo/request.py +287 -206
- vortex/nwp/algo/stdpost.py +878 -522
- vortex/nwp/data/__init__.py +22 -4
- vortex/nwp/data/assim.py +125 -137
- vortex/nwp/data/boundaries.py +121 -68
- vortex/nwp/data/climfiles.py +193 -211
- vortex/nwp/data/configfiles.py +73 -69
- vortex/nwp/data/consts.py +426 -401
- vortex/nwp/data/ctpini.py +59 -43
- vortex/nwp/data/diagnostics.py +94 -66
- vortex/nwp/data/eda.py +50 -51
- vortex/nwp/data/eps.py +195 -146
- vortex/nwp/data/executables.py +440 -434
- vortex/nwp/data/fields.py +63 -48
- vortex/nwp/data/gridfiles.py +183 -111
- vortex/nwp/data/logs.py +250 -217
- vortex/nwp/data/modelstates.py +180 -151
- vortex/nwp/data/monitoring.py +72 -99
- vortex/nwp/data/namelists.py +254 -202
- vortex/nwp/data/obs.py +400 -308
- vortex/nwp/data/oopsexec.py +22 -20
- vortex/nwp/data/providers.py +90 -65
- vortex/nwp/data/query.py +71 -82
- vortex/nwp/data/stores.py +49 -36
- vortex/nwp/data/surfex.py +136 -137
- vortex/nwp/syntax/__init__.py +1 -1
- vortex/nwp/syntax/stdattrs.py +173 -111
- vortex/nwp/tools/__init__.py +2 -2
- vortex/nwp/tools/addons.py +22 -17
- vortex/nwp/tools/agt.py +24 -12
- vortex/nwp/tools/bdap.py +16 -5
- vortex/nwp/tools/bdcp.py +4 -1
- vortex/nwp/tools/bdm.py +3 -0
- vortex/nwp/tools/bdmp.py +14 -9
- vortex/nwp/tools/conftools.py +728 -378
- vortex/nwp/tools/drhook.py +12 -8
- vortex/nwp/tools/grib.py +65 -39
- vortex/nwp/tools/gribdiff.py +22 -17
- vortex/nwp/tools/ifstools.py +82 -42
- vortex/nwp/tools/igastuff.py +167 -143
- vortex/nwp/tools/mars.py +14 -2
- vortex/nwp/tools/odb.py +234 -125
- vortex/nwp/tools/partitioning.py +61 -37
- vortex/nwp/tools/satrad.py +27 -12
- vortex/nwp/util/async.py +83 -55
- vortex/nwp/util/beacon.py +10 -10
- vortex/nwp/util/diffpygram.py +174 -86
- vortex/nwp/util/ens.py +144 -63
- vortex/nwp/util/hooks.py +30 -19
- vortex/nwp/util/taskdeco.py +28 -24
- vortex/nwp/util/usepygram.py +278 -172
- vortex/nwp/util/usetnt.py +31 -17
- vortex/sessions.py +72 -39
- vortex/syntax/__init__.py +1 -1
- vortex/syntax/stdattrs.py +410 -171
- vortex/syntax/stddeco.py +31 -22
- vortex/toolbox.py +327 -192
- vortex/tools/__init__.py +11 -2
- vortex/tools/actions.py +110 -121
- vortex/tools/addons.py +111 -92
- vortex/tools/arm.py +42 -22
- vortex/tools/compression.py +72 -69
- vortex/tools/date.py +11 -4
- vortex/tools/delayedactions.py +242 -132
- vortex/tools/env.py +75 -47
- vortex/tools/folder.py +342 -171
- vortex/tools/grib.py +341 -162
- vortex/tools/lfi.py +423 -216
- vortex/tools/listings.py +109 -40
- vortex/tools/names.py +218 -156
- vortex/tools/net.py +655 -299
- vortex/tools/parallelism.py +93 -61
- vortex/tools/prestaging.py +55 -31
- vortex/tools/schedulers.py +172 -105
- vortex/tools/services.py +403 -334
- vortex/tools/storage.py +293 -358
- vortex/tools/surfex.py +24 -24
- vortex/tools/systems.py +1234 -643
- vortex/tools/targets.py +156 -100
- vortex/util/__init__.py +1 -1
- vortex/util/config.py +378 -327
- vortex/util/empty.py +2 -2
- vortex/util/helpers.py +56 -24
- vortex/util/introspection.py +18 -12
- vortex/util/iosponge.py +8 -4
- vortex/util/roles.py +4 -6
- vortex/util/storefunctions.py +39 -13
- vortex/util/structs.py +3 -3
- vortex/util/worker.py +29 -17
- vortex_nwp-2.1.0.dist-info/METADATA +67 -0
- vortex_nwp-2.1.0.dist-info/RECORD +144 -0
- {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.1.0.dist-info}/WHEEL +1 -1
- vortex/layout/appconf.py +0 -109
- vortex/layout/jobs.py +0 -1276
- vortex/layout/nodes.py +0 -1424
- vortex/layout/subjobs.py +0 -464
- vortex_nwp-2.0.0b1.dist-info/METADATA +0 -50
- vortex_nwp-2.0.0b1.dist-info/RECORD +0 -146
- {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.1.0.dist-info/licenses}/LICENSE +0 -0
- {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.1.0.dist-info}/top_level.txt +0 -0
vortex/tools/systems.py
CHANGED
|
@@ -67,6 +67,8 @@ from vortex.tools.compression import CompressionPipeline
|
|
|
67
67
|
from vortex.tools.env import Environment
|
|
68
68
|
from vortex.tools.net import AssistedSsh, AutoRetriesFtp, DEFAULT_FTP_PORT
|
|
69
69
|
from vortex.tools.net import FtpConnectionPool, LinuxNetstats, StdFtp
|
|
70
|
+
import vortex.tools.storage
|
|
71
|
+
from vortex import config
|
|
70
72
|
|
|
71
73
|
#: No automatic export
|
|
72
74
|
__all__ = []
|
|
@@ -74,13 +76,13 @@ __all__ = []
|
|
|
74
76
|
logger = loggers.getLogger(__name__)
|
|
75
77
|
|
|
76
78
|
#: Pre-compiled regex to check a none str value
|
|
77
|
-
isnonedef = re.compile(r
|
|
79
|
+
isnonedef = re.compile(r"\s*none\s*$", re.IGNORECASE)
|
|
78
80
|
|
|
79
81
|
#: Pre-compiled regex to check a boolean true str value
|
|
80
|
-
istruedef = re.compile(r
|
|
82
|
+
istruedef = re.compile(r"\s*(on|true|ok)\s*$", re.IGNORECASE)
|
|
81
83
|
|
|
82
84
|
#: Pre-compiled regex to check a boolean false str value
|
|
83
|
-
isfalsedef = re.compile(r
|
|
85
|
+
isfalsedef = re.compile(r"\s*(off|false|ko)\s*$", re.IGNORECASE)
|
|
84
86
|
|
|
85
87
|
#: Global lock to protect temporary locale changes
|
|
86
88
|
LOCALE_LOCK = threading.Lock()
|
|
@@ -97,7 +99,9 @@ _fmtshcmd_docbonus = """
|
|
|
97
99
|
# Constant items
|
|
98
100
|
|
|
99
101
|
#: Definition of a named tuple ftpflavour
|
|
100
|
-
FtpFlavourTuple = namedtuple(
|
|
102
|
+
FtpFlavourTuple = namedtuple(
|
|
103
|
+
"FtpFlavourTuple", ["STD", "RETRIES", "CONNECTION_POOLS"]
|
|
104
|
+
)
|
|
101
105
|
|
|
102
106
|
#: Predefined FTP_FLAVOUR values IN, OUT and INOUT.
|
|
103
107
|
FTP_FLAVOUR = FtpFlavourTuple(STD=0, RETRIES=1, CONNECTION_POOLS=2)
|
|
@@ -114,10 +118,12 @@ def fmtshcmd(func):
|
|
|
114
118
|
"""
|
|
115
119
|
|
|
116
120
|
def formatted_method(self, *args, **kw):
|
|
117
|
-
fmt = kw.pop(
|
|
121
|
+
fmt = kw.pop("fmt", None)
|
|
118
122
|
shtarget = self if isinstance(self, System) else self.sh
|
|
119
|
-
fmtcall = getattr(
|
|
120
|
-
|
|
123
|
+
fmtcall = getattr(
|
|
124
|
+
shtarget, str(fmt).lower() + "_" + func.__name__, func
|
|
125
|
+
)
|
|
126
|
+
if getattr(fmtcall, "func_extern", False):
|
|
121
127
|
return fmtcall(*args, **kw)
|
|
122
128
|
else:
|
|
123
129
|
return fmtcall(self, *args, **kw)
|
|
@@ -144,11 +150,13 @@ def _kw2spawn(func):
|
|
|
144
150
|
|
|
145
151
|
class ExecutionError(RuntimeError):
|
|
146
152
|
"""Go through exception for internal :meth:`OSExtended.spawn` errors."""
|
|
153
|
+
|
|
147
154
|
pass
|
|
148
155
|
|
|
149
156
|
|
|
150
157
|
class CopyTreeError(OSError):
|
|
151
158
|
"""An error raised during the recursive copy of a directory."""
|
|
159
|
+
|
|
152
160
|
pass
|
|
153
161
|
|
|
154
162
|
|
|
@@ -171,12 +179,12 @@ class CdContext:
|
|
|
171
179
|
self.newpath = self.sh.path.expanduser(newpath)
|
|
172
180
|
|
|
173
181
|
def __enter__(self):
|
|
174
|
-
if self.newpath not in (
|
|
182
|
+
if self.newpath not in ("", "."):
|
|
175
183
|
self.oldpath = self.sh.getcwd()
|
|
176
184
|
self.sh.cd(self.newpath, create=self.create)
|
|
177
185
|
|
|
178
186
|
def __exit__(self, etype, value, traceback): # @UnusedVariable
|
|
179
|
-
if self.newpath not in (
|
|
187
|
+
if self.newpath not in ("", "."):
|
|
180
188
|
self.sh.cd(self.oldpath)
|
|
181
189
|
if self.clean_onexit:
|
|
182
190
|
self.sh.rm(self.newpath)
|
|
@@ -220,14 +228,14 @@ class PythonSimplifiedVersion:
|
|
|
220
228
|
It can be used in a footprint specification.
|
|
221
229
|
"""
|
|
222
230
|
|
|
223
|
-
_VERSION_RE = re.compile(r
|
|
231
|
+
_VERSION_RE = re.compile(r"(\d+)\.(\d+)\.(\d+)")
|
|
224
232
|
|
|
225
233
|
def __init__(self, versionstr):
|
|
226
234
|
v_match = self._VERSION_RE.match(versionstr)
|
|
227
235
|
if v_match:
|
|
228
236
|
self._version = tuple([int(d) for d in v_match.groups()])
|
|
229
237
|
else:
|
|
230
|
-
raise ValueError(
|
|
238
|
+
raise ValueError("Malformed version string: {}".format(versionstr))
|
|
231
239
|
|
|
232
240
|
@property
|
|
233
241
|
def version(self):
|
|
@@ -250,10 +258,12 @@ class PythonSimplifiedVersion:
|
|
|
250
258
|
return self.version > other.version
|
|
251
259
|
|
|
252
260
|
def __str__(self):
|
|
253
|
-
return
|
|
261
|
+
return ".".join([str(d) for d in self.version])
|
|
254
262
|
|
|
255
263
|
def __repr__(self):
|
|
256
|
-
return
|
|
264
|
+
return "<{} | {!s}>".format(
|
|
265
|
+
object.__repr__(self).lstrip("<").rstrip(">"), self
|
|
266
|
+
)
|
|
257
267
|
|
|
258
268
|
def export_dict(self):
|
|
259
269
|
"""The pure dict/json output is the raw integer"""
|
|
@@ -269,50 +279,50 @@ class System(footprints.FootprintBase):
|
|
|
269
279
|
|
|
270
280
|
_abstract = True
|
|
271
281
|
_explicit = False
|
|
272
|
-
_collector = (
|
|
282
|
+
_collector = ("system",)
|
|
273
283
|
|
|
274
284
|
_footprint = dict(
|
|
275
|
-
info
|
|
276
|
-
attr
|
|
277
|
-
hostname
|
|
278
|
-
info
|
|
279
|
-
optional
|
|
280
|
-
default
|
|
281
|
-
alias
|
|
285
|
+
info="Default system interface",
|
|
286
|
+
attr=dict(
|
|
287
|
+
hostname=dict(
|
|
288
|
+
info="The computer's network name",
|
|
289
|
+
optional=True,
|
|
290
|
+
default=platform.node(),
|
|
291
|
+
alias=("nodename",),
|
|
282
292
|
),
|
|
283
|
-
sysname
|
|
284
|
-
info
|
|
285
|
-
optional
|
|
286
|
-
default
|
|
293
|
+
sysname=dict(
|
|
294
|
+
info="The underlying system/OS name (e.g. Linux, Darwin, ...)",
|
|
295
|
+
optional=True,
|
|
296
|
+
default=platform.system(),
|
|
287
297
|
),
|
|
288
|
-
arch
|
|
289
|
-
info
|
|
290
|
-
optional
|
|
291
|
-
default
|
|
292
|
-
alias
|
|
298
|
+
arch=dict(
|
|
299
|
+
info="The underlying machine type (e.g. i386, x86_64, ...)",
|
|
300
|
+
optional=True,
|
|
301
|
+
default=platform.machine(),
|
|
302
|
+
alias=("machine",),
|
|
293
303
|
),
|
|
294
|
-
release
|
|
295
|
-
info
|
|
296
|
-
optional
|
|
297
|
-
default
|
|
304
|
+
release=dict(
|
|
305
|
+
info="The underlying system's release, (e.g. 2.2.0, NT, ...)",
|
|
306
|
+
optional=True,
|
|
307
|
+
default=platform.release(),
|
|
298
308
|
),
|
|
299
|
-
version
|
|
300
|
-
info
|
|
301
|
-
optional
|
|
302
|
-
default
|
|
309
|
+
version=dict(
|
|
310
|
+
info="The underlying system's release version",
|
|
311
|
+
optional=True,
|
|
312
|
+
default=platform.version(),
|
|
303
313
|
),
|
|
304
|
-
python
|
|
305
|
-
info
|
|
306
|
-
type
|
|
307
|
-
optional
|
|
308
|
-
default
|
|
314
|
+
python=dict(
|
|
315
|
+
info="The Python's version (e.g 2.7.5)",
|
|
316
|
+
type=PythonSimplifiedVersion,
|
|
317
|
+
optional=True,
|
|
318
|
+
default=platform.python_version(),
|
|
309
319
|
),
|
|
310
|
-
glove
|
|
311
|
-
info
|
|
312
|
-
optional
|
|
313
|
-
type
|
|
314
|
-
)
|
|
315
|
-
)
|
|
320
|
+
glove=dict(
|
|
321
|
+
info="The session's Glove object",
|
|
322
|
+
optional=True,
|
|
323
|
+
type=Glove,
|
|
324
|
+
),
|
|
325
|
+
),
|
|
316
326
|
)
|
|
317
327
|
|
|
318
328
|
def __init__(self, *args, **kw):
|
|
@@ -364,25 +374,29 @@ class System(footprints.FootprintBase):
|
|
|
364
374
|
user will be able to call ``sh.greatstuff``).
|
|
365
375
|
|
|
366
376
|
"""
|
|
367
|
-
logger.debug(
|
|
368
|
-
self.__dict__[
|
|
369
|
-
self.__dict__[
|
|
370
|
-
self.__dict__[
|
|
371
|
-
self.__dict__[
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
+
logger.debug("Abstract System init %s", self.__class__)
|
|
378
|
+
self.__dict__["_os"] = kw.pop("os", os)
|
|
379
|
+
self.__dict__["_rl"] = kw.pop("rlimit", resource)
|
|
380
|
+
self.__dict__["_sh"] = kw.pop("shutil", kw.pop("sh", shutil))
|
|
381
|
+
self.__dict__["_search"] = [
|
|
382
|
+
self.__dict__["_os"],
|
|
383
|
+
self.__dict__["_sh"],
|
|
384
|
+
self.__dict__["_rl"],
|
|
385
|
+
]
|
|
386
|
+
self.__dict__["_xtrack"] = dict()
|
|
387
|
+
self.__dict__["_history"] = History(tag="shell")
|
|
388
|
+
self.__dict__["_rclast"] = 0
|
|
389
|
+
self.__dict__["prompt"] = str(kw.pop("prompt", ""))
|
|
390
|
+
for flag in ("trace", "timer"):
|
|
377
391
|
self.__dict__[flag] = kw.pop(flag, False)
|
|
378
|
-
for flag in (
|
|
392
|
+
for flag in ("output",):
|
|
379
393
|
self.__dict__[flag] = kw.pop(flag, True)
|
|
380
394
|
super().__init__(*args, **kw)
|
|
381
395
|
|
|
382
396
|
@property
|
|
383
397
|
def realkind(self):
|
|
384
398
|
"""The object/class realkind."""
|
|
385
|
-
return
|
|
399
|
+
return "system"
|
|
386
400
|
|
|
387
401
|
@property
|
|
388
402
|
def history(self):
|
|
@@ -407,18 +421,18 @@ class System(footprints.FootprintBase):
|
|
|
407
421
|
@property
|
|
408
422
|
def default_syslog(self):
|
|
409
423
|
"""Address to use in logging.handler.SysLogHandler()."""
|
|
410
|
-
return
|
|
424
|
+
return "/dev/log"
|
|
411
425
|
|
|
412
426
|
def extend(self, obj=None):
|
|
413
427
|
"""Extend the current external attribute resolution to **obj** (module or object)."""
|
|
414
428
|
if obj is not None:
|
|
415
|
-
if hasattr(obj,
|
|
429
|
+
if hasattr(obj, "kind"):
|
|
416
430
|
for k, v in self._xtrack.items():
|
|
417
|
-
if hasattr(v,
|
|
431
|
+
if hasattr(v, "kind"):
|
|
418
432
|
if hasattr(self, k):
|
|
419
433
|
delattr(self, k)
|
|
420
434
|
for addon in self.search:
|
|
421
|
-
if hasattr(addon,
|
|
435
|
+
if hasattr(addon, "kind") and addon.kind == obj.kind:
|
|
422
436
|
self.search.remove(addon)
|
|
423
437
|
self.search.append(obj)
|
|
424
438
|
return len(self.search)
|
|
@@ -429,7 +443,7 @@ class System(footprints.FootprintBase):
|
|
|
429
443
|
(*i.e.* :class:`~vortex.tools.addons.Addon objects previously
|
|
430
444
|
loaded with the :meth:`extend` method).
|
|
431
445
|
"""
|
|
432
|
-
return [addon.kind for addon in self.search if hasattr(addon,
|
|
446
|
+
return [addon.kind for addon in self.search if hasattr(addon, "kind")]
|
|
433
447
|
|
|
434
448
|
def external(self, key):
|
|
435
449
|
"""Return effective module object reference if any, or *None*."""
|
|
@@ -446,12 +460,14 @@ class System(footprints.FootprintBase):
|
|
|
446
460
|
This is the place where the ``self.search`` list is looked for...
|
|
447
461
|
"""
|
|
448
462
|
actualattr = None
|
|
449
|
-
if key.startswith(
|
|
463
|
+
if key.startswith("_"):
|
|
450
464
|
# Do not attempt to look for hidden attributes
|
|
451
|
-
raise AttributeError(
|
|
465
|
+
raise AttributeError("Method or attribute " + key + " not found")
|
|
452
466
|
for shxobj in self.search:
|
|
453
467
|
if hasattr(shxobj, key):
|
|
454
|
-
if isinstance(
|
|
468
|
+
if isinstance(
|
|
469
|
+
shxobj, footprints.FootprintBase
|
|
470
|
+
) and shxobj.footprint_has_attribute(key):
|
|
455
471
|
# Ignore footprint attributes
|
|
456
472
|
continue
|
|
457
473
|
if actualattr is None:
|
|
@@ -459,17 +475,24 @@ class System(footprints.FootprintBase):
|
|
|
459
475
|
self._xtrack[key] = shxobj
|
|
460
476
|
else:
|
|
461
477
|
# Do not warn for a restricted list of keys
|
|
462
|
-
if key not in (
|
|
463
|
-
logger.warning(
|
|
464
|
-
|
|
465
|
-
|
|
478
|
+
if key not in ("stat",):
|
|
479
|
+
logger.warning(
|
|
480
|
+
'System: duplicate entry while looking for key="%s". '
|
|
481
|
+
+ "First result in %s but also available in %s.",
|
|
482
|
+
key,
|
|
483
|
+
self._xtrack[key],
|
|
484
|
+
shxobj,
|
|
485
|
+
)
|
|
466
486
|
if actualattr is None:
|
|
467
|
-
raise AttributeError(
|
|
487
|
+
raise AttributeError("Method or attribute " + key + " not found")
|
|
468
488
|
if callable(actualattr):
|
|
489
|
+
|
|
469
490
|
def osproxy(*args, **kw):
|
|
470
491
|
cmd = [key]
|
|
471
492
|
cmd.extend(args)
|
|
472
|
-
cmd.extend(
|
|
493
|
+
cmd.extend(
|
|
494
|
+
["{:s}={:s}".format(x, str(kw[x])) for x in kw.keys()]
|
|
495
|
+
)
|
|
473
496
|
self.stderr(*cmd)
|
|
474
497
|
return actualattr(*args, **kw)
|
|
475
498
|
|
|
@@ -484,15 +507,21 @@ class System(footprints.FootprintBase):
|
|
|
484
507
|
|
|
485
508
|
def stderr(self, *args):
|
|
486
509
|
"""Write a formatted message to standard error (if ``self.trace == True``)."""
|
|
487
|
-
|
|
510
|
+
(
|
|
511
|
+
count,
|
|
512
|
+
justnow,
|
|
513
|
+
) = self.history.append(*args)
|
|
488
514
|
if self.trace:
|
|
489
|
-
if self.trace ==
|
|
490
|
-
logger.info(
|
|
515
|
+
if self.trace == "log":
|
|
516
|
+
logger.info(
|
|
517
|
+
"[sh:#%d] %s", count, " ".join([str(x) for x in args])
|
|
518
|
+
)
|
|
491
519
|
else:
|
|
492
520
|
sys.stderr.write(
|
|
493
521
|
"* [{:s}][{:d}] {:s}\n".format(
|
|
494
|
-
justnow.strftime(
|
|
495
|
-
|
|
522
|
+
justnow.strftime("%Y/%m/%d-%H:%M:%S"),
|
|
523
|
+
count,
|
|
524
|
+
" ".join([str(x) for x in args]),
|
|
496
525
|
)
|
|
497
526
|
)
|
|
498
527
|
|
|
@@ -512,9 +541,9 @@ class System(footprints.FootprintBase):
|
|
|
512
541
|
|
|
513
542
|
def echo(self, *args):
|
|
514
543
|
"""Joined **args** are echoed."""
|
|
515
|
-
print(
|
|
544
|
+
print(">>>", " ".join([str(arg) for arg in args]))
|
|
516
545
|
|
|
517
|
-
def title(self, textlist, tchar=
|
|
546
|
+
def title(self, textlist, tchar="=", autolen=96):
|
|
518
547
|
"""Formated title output.
|
|
519
548
|
|
|
520
549
|
:param list|str textlist: A list of strings that contains the title's text
|
|
@@ -530,12 +559,16 @@ class System(footprints.FootprintBase):
|
|
|
530
559
|
print()
|
|
531
560
|
print(tchar * (nbc + 4))
|
|
532
561
|
for text in textlist:
|
|
533
|
-
print(
|
|
562
|
+
print(
|
|
563
|
+
"{0:s} {1:^{size}s} {0:s}".format(
|
|
564
|
+
tchar, text.upper(), size=nbc
|
|
565
|
+
)
|
|
566
|
+
)
|
|
534
567
|
print(tchar * (nbc + 4))
|
|
535
568
|
print()
|
|
536
569
|
self.flush_stdall()
|
|
537
570
|
|
|
538
|
-
def subtitle(self, text=
|
|
571
|
+
def subtitle(self, text="", tchar="-", autolen=96):
|
|
539
572
|
"""Formatted subtitle output.
|
|
540
573
|
|
|
541
574
|
:param str text: The subtitle's text
|
|
@@ -549,11 +582,13 @@ class System(footprints.FootprintBase):
|
|
|
549
582
|
print()
|
|
550
583
|
print(tchar * (nbc + 4))
|
|
551
584
|
if text:
|
|
552
|
-
print(
|
|
585
|
+
print("# {0:{size}s} #".format(text, size=nbc))
|
|
553
586
|
print(tchar * (nbc + 4))
|
|
554
587
|
self.flush_stdall()
|
|
555
588
|
|
|
556
|
-
def header(
|
|
589
|
+
def header(
|
|
590
|
+
self, text="", tchar="-", autolen=False, xline=True, prompt=None
|
|
591
|
+
):
|
|
557
592
|
"""Formatted header output.
|
|
558
593
|
|
|
559
594
|
:param str text: The subtitle's text
|
|
@@ -572,15 +607,17 @@ class System(footprints.FootprintBase):
|
|
|
572
607
|
if not prompt:
|
|
573
608
|
prompt = self.prompt
|
|
574
609
|
if prompt:
|
|
575
|
-
prompt = str(prompt) +
|
|
610
|
+
prompt = str(prompt) + " "
|
|
576
611
|
else:
|
|
577
|
-
prompt =
|
|
612
|
+
prompt = ""
|
|
578
613
|
print(prompt + str(text))
|
|
579
614
|
if xline:
|
|
580
615
|
print(tchar * nbc)
|
|
581
616
|
self.flush_stdall()
|
|
582
617
|
|
|
583
|
-
def highlight(
|
|
618
|
+
def highlight(
|
|
619
|
+
self, text="", hchar="----", bchar="#", bline=False, bline0=True
|
|
620
|
+
):
|
|
584
621
|
"""Highlight some text.
|
|
585
622
|
|
|
586
623
|
:param str text: The text to be highlighted
|
|
@@ -591,8 +628,11 @@ class System(footprints.FootprintBase):
|
|
|
591
628
|
"""
|
|
592
629
|
if bline0:
|
|
593
630
|
print()
|
|
594
|
-
print(
|
|
595
|
-
|
|
631
|
+
print(
|
|
632
|
+
"{0:s} {1:s} {2:s} {1:s} {3:s}".format(
|
|
633
|
+
bchar.rstrip(), hchar, text, bchar.lstrip()
|
|
634
|
+
)
|
|
635
|
+
)
|
|
596
636
|
if bline:
|
|
597
637
|
print()
|
|
598
638
|
self.flush_stdall()
|
|
@@ -600,18 +640,18 @@ class System(footprints.FootprintBase):
|
|
|
600
640
|
@property
|
|
601
641
|
def executable(self):
|
|
602
642
|
"""Return the actual ``sys.executable``."""
|
|
603
|
-
self.stderr(
|
|
643
|
+
self.stderr("executable")
|
|
604
644
|
return sys.executable
|
|
605
645
|
|
|
606
646
|
def pythonpath(self, output=None):
|
|
607
647
|
"""Return or print actual ``sys.path``."""
|
|
608
648
|
if output is None:
|
|
609
649
|
output = self.output
|
|
610
|
-
self.stderr(
|
|
650
|
+
self.stderr("pythonpath")
|
|
611
651
|
if output:
|
|
612
652
|
return sys.path[:]
|
|
613
653
|
else:
|
|
614
|
-
self.subtitle(
|
|
654
|
+
self.subtitle("Python PATH")
|
|
615
655
|
for pypath in sys.path:
|
|
616
656
|
print(pypath)
|
|
617
657
|
return True
|
|
@@ -625,12 +665,12 @@ class System(footprints.FootprintBase):
|
|
|
625
665
|
"""Try to determine an identification string for the current script."""
|
|
626
666
|
# PBS scheduler SLURM scheduler Good-old PID
|
|
627
667
|
env = self.env
|
|
628
|
-
label = env.PBS_JOBID or env.SLURM_JOB_ID or
|
|
629
|
-
if label ==
|
|
668
|
+
label = env.PBS_JOBID or env.SLURM_JOB_ID or "localpid"
|
|
669
|
+
if label == "localpid":
|
|
630
670
|
label = str(self.getpid())
|
|
631
671
|
return label
|
|
632
672
|
|
|
633
|
-
def vortex_modules(self, only=
|
|
673
|
+
def vortex_modules(self, only="."):
|
|
634
674
|
"""Return a filtered list of modules in the vortex package.
|
|
635
675
|
|
|
636
676
|
:param str only: The regex used to filter the modules list.
|
|
@@ -638,22 +678,26 @@ class System(footprints.FootprintBase):
|
|
|
638
678
|
if self.glove is not None:
|
|
639
679
|
g = self.glove
|
|
640
680
|
mfiles = [
|
|
641
|
-
re.sub(r
|
|
642
|
-
for mroot in (g.siteroot +
|
|
681
|
+
re.sub(r"^" + mroot + r"/", "", x)
|
|
682
|
+
for mroot in (g.siteroot + "/src", g.siteroot + "/site")
|
|
643
683
|
for x in self.ffind(mroot)
|
|
644
|
-
if self.path.isfile(
|
|
684
|
+
if self.path.isfile(
|
|
685
|
+
self.path.join(self.path.dirname(x), "__init__.py")
|
|
686
|
+
)
|
|
645
687
|
]
|
|
646
688
|
return [
|
|
647
|
-
re.sub(r
|
|
689
|
+
re.sub(r"(?:/__init__)?\.py$", "", x).replace("/", ".")
|
|
648
690
|
for x in mfiles
|
|
649
|
-
if (
|
|
650
|
-
|
|
651
|
-
|
|
691
|
+
if (
|
|
692
|
+
not x.startswith(".")
|
|
693
|
+
and re.search(only, x, re.IGNORECASE)
|
|
694
|
+
and x.endswith(".py")
|
|
695
|
+
)
|
|
652
696
|
]
|
|
653
697
|
else:
|
|
654
698
|
raise RuntimeError("A glove must be defined")
|
|
655
699
|
|
|
656
|
-
def vortex_loaded_modules(self, only=
|
|
700
|
+
def vortex_loaded_modules(self, only=".", output=None):
|
|
657
701
|
"""Check loaded modules, producing either a dump or a list of tuple (status, modulename).
|
|
658
702
|
|
|
659
703
|
:param str only: The regex used to filter the modules list.
|
|
@@ -666,7 +710,7 @@ class System(footprints.FootprintBase):
|
|
|
666
710
|
if not output:
|
|
667
711
|
for m, s in checklist:
|
|
668
712
|
print(str(s).ljust(8), m)
|
|
669
|
-
print(
|
|
713
|
+
print("--")
|
|
670
714
|
return True
|
|
671
715
|
else:
|
|
672
716
|
return checklist
|
|
@@ -674,13 +718,17 @@ class System(footprints.FootprintBase):
|
|
|
674
718
|
def systems_reload(self):
|
|
675
719
|
"""Load extra systems modules not yet loaded."""
|
|
676
720
|
extras = list()
|
|
677
|
-
for modname in self.vortex_modules(only=
|
|
721
|
+
for modname in self.vortex_modules(only="systems"):
|
|
678
722
|
if modname not in sys.modules:
|
|
679
723
|
try:
|
|
680
724
|
self.import_module(modname)
|
|
681
725
|
extras.append(modname)
|
|
682
726
|
except ValueError as err:
|
|
683
|
-
logger.critical(
|
|
727
|
+
logger.critical(
|
|
728
|
+
"systems_reload: cannot import module %s (%s)",
|
|
729
|
+
modname,
|
|
730
|
+
str(err),
|
|
731
|
+
)
|
|
684
732
|
return extras
|
|
685
733
|
|
|
686
734
|
# Redefinition of methods of the resource package...
|
|
@@ -692,16 +740,16 @@ class System(footprints.FootprintBase):
|
|
|
692
740
|
"""
|
|
693
741
|
if type(r_id) is not int:
|
|
694
742
|
r_id = r_id.upper()
|
|
695
|
-
if not r_id.startswith(
|
|
696
|
-
r_id =
|
|
743
|
+
if not r_id.startswith("RLIMIT_"):
|
|
744
|
+
r_id = "RLIMIT_" + r_id
|
|
697
745
|
r_id = getattr(self._rl, r_id, None)
|
|
698
746
|
if r_id is None:
|
|
699
|
-
raise ValueError(
|
|
747
|
+
raise ValueError("Invalid resource specified")
|
|
700
748
|
return r_id
|
|
701
749
|
|
|
702
750
|
def setrlimit(self, r_id, r_limits):
|
|
703
751
|
"""Proxy to :mod:`resource` function of the same name."""
|
|
704
|
-
self.stderr(
|
|
752
|
+
self.stderr("setrlimit", r_id, r_limits)
|
|
705
753
|
try:
|
|
706
754
|
r_limits = tuple(r_limits)
|
|
707
755
|
except TypeError:
|
|
@@ -710,14 +758,14 @@ class System(footprints.FootprintBase):
|
|
|
710
758
|
|
|
711
759
|
def getrlimit(self, r_id):
|
|
712
760
|
"""Proxy to :mod:`resource` function of the same name."""
|
|
713
|
-
self.stderr(
|
|
761
|
+
self.stderr("getrlimit", r_id)
|
|
714
762
|
return self._rl.getrlimit(self.numrlimit(r_id))
|
|
715
763
|
|
|
716
764
|
def getrusage(self, pid=None):
|
|
717
765
|
"""Proxy to :mod:`resource` function of the same name with current process as defaut."""
|
|
718
766
|
if pid is None:
|
|
719
767
|
pid = self._rl.RUSAGE_SELF
|
|
720
|
-
self.stderr(
|
|
768
|
+
self.stderr("getrusage", pid)
|
|
721
769
|
return self._rl.getrusage(pid)
|
|
722
770
|
|
|
723
771
|
def import_module(self, modname):
|
|
@@ -728,12 +776,12 @@ class System(footprints.FootprintBase):
|
|
|
728
776
|
def import_function(self, funcname):
|
|
729
777
|
"""Import the function named **funcname** qualified by a proper module name package."""
|
|
730
778
|
thisfunc = None
|
|
731
|
-
if
|
|
732
|
-
thismod = self.import_module(
|
|
779
|
+
if "." in funcname:
|
|
780
|
+
thismod = self.import_module(".".join(funcname.split(".")[:-1]))
|
|
733
781
|
if thismod:
|
|
734
|
-
thisfunc = getattr(thismod, funcname.split(
|
|
782
|
+
thisfunc = getattr(thismod, funcname.split(".")[-1], None)
|
|
735
783
|
else:
|
|
736
|
-
logger.error(
|
|
784
|
+
logger.error("Bad function path name <%s>" % funcname)
|
|
737
785
|
return thisfunc
|
|
738
786
|
|
|
739
787
|
|
|
@@ -744,9 +792,7 @@ class OSExtended(System):
|
|
|
744
792
|
"""
|
|
745
793
|
|
|
746
794
|
_abstract = True
|
|
747
|
-
_footprint = dict(
|
|
748
|
-
info = 'Abstract extended base system'
|
|
749
|
-
)
|
|
795
|
+
_footprint = dict(info="Abstract extended base system")
|
|
750
796
|
|
|
751
797
|
def __init__(self, *args, **kw):
|
|
752
798
|
"""
|
|
@@ -755,7 +801,7 @@ class OSExtended(System):
|
|
|
755
801
|
|
|
756
802
|
* **rmtreemin** - as the minimal depth needed for a :meth:`rmsafe`.
|
|
757
803
|
* **cmpaftercp** - as a boolean for activating full comparison after plain cp (default: *True*).
|
|
758
|
-
* **
|
|
804
|
+
* **ftserv** - allows ``smartft*`` methods to use the raw FTP commands
|
|
759
805
|
(e.g. ftget, ftput) instead of the internal Vortex's FTP client
|
|
760
806
|
(default: *False*).
|
|
761
807
|
* **ftputcmd** - The name of the raw FTP command for the "put" action
|
|
@@ -766,15 +812,15 @@ class OSExtended(System):
|
|
|
766
812
|
(default: `FTP_FLAVOUR.CONNECTION_POOLS`). See the :meth:`ftp` method
|
|
767
813
|
for more details.
|
|
768
814
|
"""
|
|
769
|
-
logger.debug(
|
|
770
|
-
self._rmtreemin = kw.pop(
|
|
771
|
-
self._cmpaftercp = kw.pop(
|
|
815
|
+
logger.debug("Abstract System init %s", self.__class__)
|
|
816
|
+
self._rmtreemin = kw.pop("rmtreemin", 3)
|
|
817
|
+
self._cmpaftercp = kw.pop("cmpaftercp", True)
|
|
772
818
|
# Switches for rawft* methods
|
|
773
|
-
self.
|
|
774
|
-
self.ftputcmd = kw.pop(
|
|
775
|
-
self.ftgetcmd = kw.pop(
|
|
819
|
+
self._ftserv = kw.pop("ftserv", None)
|
|
820
|
+
self.ftputcmd = kw.pop("ftputcmd", None)
|
|
821
|
+
self.ftgetcmd = kw.pop("ftgetcmd", None)
|
|
776
822
|
# FTP stuff again
|
|
777
|
-
self.ftpflavour = kw.pop(
|
|
823
|
+
self.ftpflavour = kw.pop("ftpflavour", FTP_FLAVOUR.CONNECTION_POOLS)
|
|
778
824
|
self._current_ftppool = None
|
|
779
825
|
# Some internal variables used by particular methods
|
|
780
826
|
self._ftspool_cache = None
|
|
@@ -784,32 +830,42 @@ class OSExtended(System):
|
|
|
784
830
|
# Go for the superclass' constructor
|
|
785
831
|
super().__init__(*args, **kw)
|
|
786
832
|
# Initialise possibly missing objects
|
|
787
|
-
self.__dict__[
|
|
788
|
-
self.__dict__[
|
|
789
|
-
self.__dict__[
|
|
790
|
-
self.__dict__[
|
|
833
|
+
self.__dict__["_cpusinfo"] = None
|
|
834
|
+
self.__dict__["_numainfo"] = None
|
|
835
|
+
self.__dict__["_memoryinfo"] = None
|
|
836
|
+
self.__dict__["_netstatsinfo"] = None
|
|
791
837
|
|
|
792
838
|
# Initialise the signal handler object
|
|
793
839
|
self._signal_intercept_init()
|
|
794
840
|
|
|
795
841
|
@property
|
|
796
|
-
def
|
|
842
|
+
def ftserv(self):
|
|
797
843
|
"""Use the system's FTP service (e.g. ftserv)."""
|
|
798
|
-
if self.
|
|
799
|
-
return self.
|
|
844
|
+
if self._ftserv is None:
|
|
845
|
+
return self._use_ftserv()
|
|
800
846
|
else:
|
|
801
|
-
return self.
|
|
847
|
+
return self._ftserv
|
|
802
848
|
|
|
803
|
-
@
|
|
804
|
-
def
|
|
849
|
+
@ftserv.setter
|
|
850
|
+
def ftserv(self, value):
|
|
805
851
|
"""Use the system's FTP service (e.g. ftserv)."""
|
|
806
852
|
self._ftraw = bool(value)
|
|
807
853
|
|
|
808
|
-
@
|
|
809
|
-
def
|
|
854
|
+
@ftserv.deleter
|
|
855
|
+
def ftserv(self):
|
|
810
856
|
"""Use the system's FTP service (e.g. ftserv)."""
|
|
811
857
|
self._ftraw = None
|
|
812
858
|
|
|
859
|
+
def _use_ftserv(self):
|
|
860
|
+
if not config.is_defined(section="ftserv"):
|
|
861
|
+
return False
|
|
862
|
+
for rgxp in config.from_config(
|
|
863
|
+
section="ftserv", key="hostname_patterns"
|
|
864
|
+
):
|
|
865
|
+
if re.match(rgxp, self.hostname):
|
|
866
|
+
return True
|
|
867
|
+
return False
|
|
868
|
+
|
|
813
869
|
def target(self, **kw):
|
|
814
870
|
"""
|
|
815
871
|
Provide a default :class:`~vortex.tools.targets.Target` according
|
|
@@ -819,10 +875,7 @@ class OSExtended(System):
|
|
|
819
875
|
* The object returned by this method will be used in subsequent calls
|
|
820
876
|
to ::attr:`default_target` (this is the concept of frozen target).
|
|
821
877
|
"""
|
|
822
|
-
desc = dict(
|
|
823
|
-
hostname=self.hostname,
|
|
824
|
-
sysname=self.sysname
|
|
825
|
-
)
|
|
878
|
+
desc = dict(hostname=self.hostname, sysname=self.sysname)
|
|
826
879
|
desc.update(kw)
|
|
827
880
|
self._frozen_target = footprints.proxy.targets.default(**desc)
|
|
828
881
|
return self._frozen_target
|
|
@@ -834,10 +887,18 @@ class OSExtended(System):
|
|
|
834
887
|
|
|
835
888
|
def fmtspecific_mtd(self, method, fmt):
|
|
836
889
|
"""Check if a format specific implementation is available for a given format."""
|
|
837
|
-
return hasattr(self,
|
|
838
|
-
|
|
839
|
-
def popen(
|
|
840
|
-
|
|
890
|
+
return hasattr(self, "{:s}_{:s}".format(fmt, method))
|
|
891
|
+
|
|
892
|
+
def popen(
|
|
893
|
+
self,
|
|
894
|
+
args,
|
|
895
|
+
stdin=None,
|
|
896
|
+
stdout=None,
|
|
897
|
+
stderr=None,
|
|
898
|
+
shell=False,
|
|
899
|
+
output=False,
|
|
900
|
+
bufsize=0,
|
|
901
|
+
): # @UnusedVariable
|
|
841
902
|
"""Return an open pipe for the **args** command.
|
|
842
903
|
|
|
843
904
|
:param str|list args: The command (+ its command-line arguments) to be
|
|
@@ -882,7 +943,14 @@ class OSExtended(System):
|
|
|
882
943
|
stdin = subprocess.PIPE
|
|
883
944
|
if stderr is True:
|
|
884
945
|
stderr = subprocess.PIPE
|
|
885
|
-
return subprocess.Popen(
|
|
946
|
+
return subprocess.Popen(
|
|
947
|
+
args,
|
|
948
|
+
bufsize=bufsize,
|
|
949
|
+
stdin=stdin,
|
|
950
|
+
stdout=stdout,
|
|
951
|
+
stderr=stderr,
|
|
952
|
+
shell=shell,
|
|
953
|
+
)
|
|
886
954
|
|
|
887
955
|
def pclose(self, p, ok=None):
|
|
888
956
|
"""Do its best to nicely shutdown the process started by **p**.
|
|
@@ -905,7 +973,7 @@ class OSExtended(System):
|
|
|
905
973
|
p.terminate()
|
|
906
974
|
except OSError as e:
|
|
907
975
|
if e.errno == 3:
|
|
908
|
-
logger.debug(
|
|
976
|
+
logger.debug("Processus %s alreaded terminated." % str(p))
|
|
909
977
|
else:
|
|
910
978
|
raise
|
|
911
979
|
|
|
@@ -917,9 +985,21 @@ class OSExtended(System):
|
|
|
917
985
|
else:
|
|
918
986
|
return False
|
|
919
987
|
|
|
920
|
-
def spawn(
|
|
921
|
-
|
|
922
|
-
|
|
988
|
+
def spawn(
|
|
989
|
+
self,
|
|
990
|
+
args,
|
|
991
|
+
ok=None,
|
|
992
|
+
shell=False,
|
|
993
|
+
stdin=None,
|
|
994
|
+
output=None,
|
|
995
|
+
outmode="a+b",
|
|
996
|
+
outsplit=True,
|
|
997
|
+
silent=False,
|
|
998
|
+
fatal=True,
|
|
999
|
+
taskset=None,
|
|
1000
|
+
taskset_id=0,
|
|
1001
|
+
taskset_bsize=1,
|
|
1002
|
+
):
|
|
923
1003
|
"""Subprocess call of **args**.
|
|
924
1004
|
|
|
925
1005
|
:param str|list[str] args: The command (+ its command-line arguments) to be
|
|
@@ -986,25 +1066,25 @@ class OSExtended(System):
|
|
|
986
1066
|
stdin = subprocess.PIPE
|
|
987
1067
|
localenv = self._os.environ.copy()
|
|
988
1068
|
if taskset is not None:
|
|
989
|
-
taskset_def = taskset.split(
|
|
990
|
-
taskset, taskset_cmd, taskset_env = self.cpus_affinity_get(
|
|
991
|
-
|
|
992
|
-
|
|
1069
|
+
taskset_def = taskset.split("_")
|
|
1070
|
+
taskset, taskset_cmd, taskset_env = self.cpus_affinity_get(
|
|
1071
|
+
taskset_id, taskset_bsize, *taskset_def
|
|
1072
|
+
)
|
|
993
1073
|
if taskset:
|
|
994
1074
|
localenv.update(taskset_env)
|
|
995
1075
|
else:
|
|
996
1076
|
logger.warning("CPU binding is not available on this platform")
|
|
997
1077
|
if isinstance(args, str):
|
|
998
1078
|
if taskset:
|
|
999
|
-
args = taskset_cmd +
|
|
1079
|
+
args = taskset_cmd + " " + args
|
|
1000
1080
|
if self.timer:
|
|
1001
|
-
args =
|
|
1081
|
+
args = "time " + args
|
|
1002
1082
|
self.stderr(args)
|
|
1003
1083
|
else:
|
|
1004
1084
|
if taskset:
|
|
1005
1085
|
args[:0] = taskset_cmd
|
|
1006
1086
|
if self.timer:
|
|
1007
|
-
args[:0] = [
|
|
1087
|
+
args[:0] = ["time"]
|
|
1008
1088
|
self.stderr(*args)
|
|
1009
1089
|
if isinstance(output, bool):
|
|
1010
1090
|
if output:
|
|
@@ -1017,36 +1097,47 @@ class OSExtended(System):
|
|
|
1017
1097
|
cmdout, cmderr = output, output
|
|
1018
1098
|
p = None
|
|
1019
1099
|
try:
|
|
1020
|
-
p = subprocess.Popen(
|
|
1021
|
-
|
|
1100
|
+
p = subprocess.Popen(
|
|
1101
|
+
args,
|
|
1102
|
+
stdin=stdin,
|
|
1103
|
+
stdout=cmdout,
|
|
1104
|
+
stderr=cmderr,
|
|
1105
|
+
shell=shell,
|
|
1106
|
+
env=localenv,
|
|
1107
|
+
)
|
|
1022
1108
|
p_out, p_err = p.communicate()
|
|
1023
1109
|
except ValueError as e:
|
|
1024
1110
|
logger.critical(
|
|
1025
|
-
|
|
1111
|
+
"Weird arguments to Popen ({!s}, stdout={!s}, stderr={!s}, shell={!s})".format(
|
|
1026
1112
|
args, cmdout, cmderr, shell
|
|
1027
1113
|
)
|
|
1028
1114
|
)
|
|
1029
|
-
logger.critical(
|
|
1115
|
+
logger.critical("Caught exception: %s", e)
|
|
1030
1116
|
if fatal:
|
|
1031
1117
|
raise
|
|
1032
1118
|
else:
|
|
1033
|
-
logger.warning(
|
|
1119
|
+
logger.warning("Carry on because fatal is off")
|
|
1034
1120
|
except OSError:
|
|
1035
|
-
logger.critical(
|
|
1121
|
+
logger.critical("Could not call %s", str(args))
|
|
1036
1122
|
if fatal:
|
|
1037
1123
|
raise
|
|
1038
1124
|
else:
|
|
1039
|
-
logger.warning(
|
|
1125
|
+
logger.warning("Carry on because fatal is off")
|
|
1040
1126
|
except Exception as perr:
|
|
1041
|
-
logger.critical(
|
|
1127
|
+
logger.critical("System returns %s", str(perr))
|
|
1042
1128
|
if fatal:
|
|
1043
|
-
raise RuntimeError(
|
|
1044
|
-
|
|
1129
|
+
raise RuntimeError(
|
|
1130
|
+
"System {!s} spawned {!s} got [{!s}]: {!s}".format(
|
|
1131
|
+
self, args, p.returncode, perr
|
|
1132
|
+
)
|
|
1133
|
+
)
|
|
1045
1134
|
else:
|
|
1046
|
-
logger.warning(
|
|
1135
|
+
logger.warning("Carry on because fatal is off")
|
|
1047
1136
|
except (SignalInterruptError, KeyboardInterrupt) as perr:
|
|
1048
|
-
logger.critical(
|
|
1049
|
-
|
|
1137
|
+
logger.critical(
|
|
1138
|
+
"The python process was killed: %s. Trying to terminate the subprocess.",
|
|
1139
|
+
str(perr),
|
|
1140
|
+
)
|
|
1050
1141
|
if p:
|
|
1051
1142
|
if shell:
|
|
1052
1143
|
# Kill the process group: apparently it's the only way when shell=T
|
|
@@ -1056,24 +1147,26 @@ class OSExtended(System):
|
|
|
1056
1147
|
p.wait()
|
|
1057
1148
|
raise # Fatal has no effect on that !
|
|
1058
1149
|
else:
|
|
1059
|
-
plocale = locale.getlocale()[1] or
|
|
1150
|
+
plocale = locale.getlocale()[1] or "ascii"
|
|
1060
1151
|
if p.returncode in ok:
|
|
1061
1152
|
if isinstance(output, bool) and output:
|
|
1062
|
-
rc = p_out.decode(plocale,
|
|
1153
|
+
rc = p_out.decode(plocale, "replace")
|
|
1063
1154
|
if outsplit:
|
|
1064
|
-
rc = rc.rstrip(
|
|
1155
|
+
rc = rc.rstrip("\n").split("\n")
|
|
1065
1156
|
p.stdout.close()
|
|
1066
1157
|
else:
|
|
1067
1158
|
rc = not bool(p.returncode)
|
|
1068
1159
|
else:
|
|
1069
1160
|
if not silent:
|
|
1070
|
-
logger.warning(
|
|
1161
|
+
logger.warning(
|
|
1162
|
+
"Bad return code [%d] for %s", p.returncode, str(args)
|
|
1163
|
+
)
|
|
1071
1164
|
if isinstance(output, bool) and output:
|
|
1072
|
-
sys.stderr.write(p_err.decode(plocale,
|
|
1165
|
+
sys.stderr.write(p_err.decode(plocale, "replace"))
|
|
1073
1166
|
if fatal:
|
|
1074
1167
|
raise ExecutionError()
|
|
1075
1168
|
else:
|
|
1076
|
-
logger.warning(
|
|
1169
|
+
logger.warning("Carry on because fatal is off")
|
|
1077
1170
|
finally:
|
|
1078
1171
|
self._rclast = p.returncode if p else 1
|
|
1079
1172
|
if isinstance(output, bool) and p:
|
|
@@ -1107,11 +1200,11 @@ class OSExtended(System):
|
|
|
1107
1200
|
"""Current working directory."""
|
|
1108
1201
|
if output is None:
|
|
1109
1202
|
output = self.output
|
|
1110
|
-
self.stderr(
|
|
1203
|
+
self.stderr("pwd")
|
|
1111
1204
|
try:
|
|
1112
1205
|
realpwd = self._os.getcwd()
|
|
1113
1206
|
except OSError as e:
|
|
1114
|
-
logger.error(
|
|
1207
|
+
logger.error("getcwdu failed: %s.", str(e))
|
|
1115
1208
|
return None
|
|
1116
1209
|
if output:
|
|
1117
1210
|
return realpwd
|
|
@@ -1122,7 +1215,7 @@ class OSExtended(System):
|
|
|
1122
1215
|
def cd(self, pathtogo, create=False):
|
|
1123
1216
|
"""Change the current working directory to **pathtogo**."""
|
|
1124
1217
|
pathtogo = self.path.expanduser(pathtogo)
|
|
1125
|
-
self.stderr(
|
|
1218
|
+
self.stderr("cd", pathtogo, create)
|
|
1126
1219
|
if create:
|
|
1127
1220
|
self.mkdir(pathtogo)
|
|
1128
1221
|
self._os.chdir(pathtogo)
|
|
@@ -1143,11 +1236,13 @@ class OSExtended(System):
|
|
|
1143
1236
|
:param prefix: The temporary directory name will start with that suffix
|
|
1144
1237
|
:param dir: The temporary directory will be created in that directory
|
|
1145
1238
|
"""
|
|
1146
|
-
self.stderr(
|
|
1147
|
-
self.stderr(
|
|
1148
|
-
with tempfile.TemporaryDirectory(
|
|
1239
|
+
self.stderr("temporary_dir_context starts", suffix)
|
|
1240
|
+
self.stderr("tempfile.TemporaryDirectory", suffix, prefix, dir)
|
|
1241
|
+
with tempfile.TemporaryDirectory(
|
|
1242
|
+
suffix=suffix, prefix=prefix, dir=dir
|
|
1243
|
+
) as tmp_dir:
|
|
1149
1244
|
yield tmp_dir
|
|
1150
|
-
self.stderr(
|
|
1245
|
+
self.stderr("tempfile.TemporaryDirectory cleanup", tmp_dir)
|
|
1151
1246
|
|
|
1152
1247
|
@contextlib.contextmanager
|
|
1153
1248
|
def temporary_dir_cdcontext(self, suffix=None, prefix=None, dir=None):
|
|
@@ -1155,23 +1250,27 @@ class OSExtended(System):
|
|
|
1155
1250
|
|
|
1156
1251
|
For a description of the context's arguments, see :func:`temporary_dir_context`.
|
|
1157
1252
|
"""
|
|
1158
|
-
with self.temporary_dir_context(
|
|
1253
|
+
with self.temporary_dir_context(
|
|
1254
|
+
suffix=suffix, prefix=prefix, dir=dir
|
|
1255
|
+
) as tmp_dir:
|
|
1159
1256
|
with self.cdcontext(tmp_dir, create=False, clean_onexit=False):
|
|
1160
1257
|
yield tmp_dir
|
|
1161
1258
|
|
|
1162
1259
|
def ffind(self, *args):
|
|
1163
1260
|
"""Recursive file find. Arguments are starting paths."""
|
|
1164
1261
|
if not args:
|
|
1165
|
-
args = [
|
|
1262
|
+
args = ["*"]
|
|
1166
1263
|
else:
|
|
1167
1264
|
args = [self.path.expanduser(x) for x in args]
|
|
1168
1265
|
files = []
|
|
1169
|
-
self.stderr(
|
|
1266
|
+
self.stderr("ffind", *args)
|
|
1170
1267
|
for pathtogo in self.glob(*args):
|
|
1171
1268
|
if self.path.isfile(pathtogo):
|
|
1172
1269
|
files.append(pathtogo)
|
|
1173
1270
|
else:
|
|
1174
|
-
for root, u_dirs, filenames in self._os.walk(
|
|
1271
|
+
for root, u_dirs, filenames in self._os.walk(
|
|
1272
|
+
pathtogo
|
|
1273
|
+
): # @UnusedVariable
|
|
1175
1274
|
files.extend([self.path.join(root, f) for f in filenames])
|
|
1176
1275
|
return sorted(files)
|
|
1177
1276
|
|
|
@@ -1184,8 +1283,13 @@ class OSExtended(System):
|
|
|
1184
1283
|
if self._os.path.exists(filename):
|
|
1185
1284
|
is_x = bool(self._os.stat(filename).st_mode & 1)
|
|
1186
1285
|
if not is_x and force:
|
|
1187
|
-
self.chmod(
|
|
1188
|
-
|
|
1286
|
+
self.chmod(
|
|
1287
|
+
filename,
|
|
1288
|
+
self._os.stat(filename).st_mode
|
|
1289
|
+
| stat.S_IXUSR
|
|
1290
|
+
| stat.S_IXGRP
|
|
1291
|
+
| stat.S_IXOTH,
|
|
1292
|
+
)
|
|
1189
1293
|
is_x = True
|
|
1190
1294
|
return is_x
|
|
1191
1295
|
else:
|
|
@@ -1199,9 +1303,16 @@ class OSExtended(System):
|
|
|
1199
1303
|
"""
|
|
1200
1304
|
if self._os.path.exists(filename):
|
|
1201
1305
|
mode = self._os.stat(filename).st_mode
|
|
1202
|
-
is_r = all(
|
|
1306
|
+
is_r = all(
|
|
1307
|
+
[
|
|
1308
|
+
bool(mode & i)
|
|
1309
|
+
for i in [stat.S_IRUSR, stat.S_IRGRP, stat.S_IROTH]
|
|
1310
|
+
]
|
|
1311
|
+
)
|
|
1203
1312
|
if not is_r and force:
|
|
1204
|
-
self.chmod(
|
|
1313
|
+
self.chmod(
|
|
1314
|
+
filename, mode | stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH
|
|
1315
|
+
)
|
|
1205
1316
|
is_r = True
|
|
1206
1317
|
return is_r
|
|
1207
1318
|
else:
|
|
@@ -1238,7 +1349,7 @@ class OSExtended(System):
|
|
|
1238
1349
|
def readonly(self, inodename):
|
|
1239
1350
|
"""Set permissions of the ``inodename`` object to read-only."""
|
|
1240
1351
|
inodename = self.path.expanduser(inodename)
|
|
1241
|
-
self.stderr(
|
|
1352
|
+
self.stderr("readonly", inodename)
|
|
1242
1353
|
rc = None
|
|
1243
1354
|
if self._os.path.exists(inodename):
|
|
1244
1355
|
if self._os.path.isdir(inodename):
|
|
@@ -1246,7 +1357,10 @@ class OSExtended(System):
|
|
|
1246
1357
|
else:
|
|
1247
1358
|
st = self.stat(inodename).st_mode
|
|
1248
1359
|
if st & stat.S_IWUSR or st & stat.S_IWGRP or st & stat.S_IWOTH:
|
|
1249
|
-
rc = self.chmod(
|
|
1360
|
+
rc = self.chmod(
|
|
1361
|
+
inodename,
|
|
1362
|
+
st & ~(stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH),
|
|
1363
|
+
)
|
|
1250
1364
|
else:
|
|
1251
1365
|
rc = True
|
|
1252
1366
|
return rc
|
|
@@ -1266,7 +1380,7 @@ class OSExtended(System):
|
|
|
1266
1380
|
def touch(self, filename):
|
|
1267
1381
|
"""Clone of the eponymous unix command."""
|
|
1268
1382
|
filename = self.path.expanduser(filename)
|
|
1269
|
-
self.stderr(
|
|
1383
|
+
self.stderr("touch", filename)
|
|
1270
1384
|
rc = True
|
|
1271
1385
|
if self.path.exists(filename):
|
|
1272
1386
|
# Note: "filename" might as well be a directory...
|
|
@@ -1275,7 +1389,7 @@ class OSExtended(System):
|
|
|
1275
1389
|
except Exception:
|
|
1276
1390
|
rc = False
|
|
1277
1391
|
else:
|
|
1278
|
-
fh = open(filename,
|
|
1392
|
+
fh = open(filename, "a")
|
|
1279
1393
|
fh.close()
|
|
1280
1394
|
return rc
|
|
1281
1395
|
|
|
@@ -1287,13 +1401,13 @@ class OSExtended(System):
|
|
|
1287
1401
|
"""
|
|
1288
1402
|
objpath = self.path.expanduser(objpath)
|
|
1289
1403
|
if self._os.path.exists(objpath):
|
|
1290
|
-
self.stderr(
|
|
1404
|
+
self.stderr("remove", objpath)
|
|
1291
1405
|
if self._os.path.isdir(objpath):
|
|
1292
1406
|
self.rmtree(objpath)
|
|
1293
1407
|
else:
|
|
1294
1408
|
self.unlink(objpath)
|
|
1295
1409
|
else:
|
|
1296
|
-
self.stderr(
|
|
1410
|
+
self.stderr("clear", objpath)
|
|
1297
1411
|
return not self._os.path.exists(objpath)
|
|
1298
1412
|
|
|
1299
1413
|
@fmtshcmd
|
|
@@ -1311,35 +1425,48 @@ class OSExtended(System):
|
|
|
1311
1425
|
expressions are used).
|
|
1312
1426
|
"""
|
|
1313
1427
|
if not pscmd:
|
|
1314
|
-
pscmd = [
|
|
1428
|
+
pscmd = ["ps"]
|
|
1315
1429
|
if opts is None:
|
|
1316
1430
|
opts = []
|
|
1317
1431
|
pscmd.extend(self._psopts)
|
|
1318
1432
|
pscmd.extend(opts)
|
|
1319
1433
|
self.stderr(*pscmd)
|
|
1320
|
-
psall =
|
|
1434
|
+
psall = (
|
|
1435
|
+
subprocess.Popen(pscmd, stdout=subprocess.PIPE)
|
|
1436
|
+
.communicate()[0]
|
|
1437
|
+
.split("\n")
|
|
1438
|
+
)
|
|
1321
1439
|
if search:
|
|
1322
1440
|
psall = filter(lambda x: re.search(search, x), psall)
|
|
1323
1441
|
return [x.strip() for x in psall]
|
|
1324
1442
|
|
|
1325
1443
|
def sleep(self, nbsecs):
|
|
1326
1444
|
"""Clone of the unix eponymous command."""
|
|
1327
|
-
self.stderr(
|
|
1445
|
+
self.stderr("sleep", nbsecs)
|
|
1328
1446
|
time.sleep(nbsecs)
|
|
1329
1447
|
|
|
1330
1448
|
def setulimit(self, r_id):
|
|
1331
1449
|
"""Set an unlimited value to the specified resource (**r_id**)."""
|
|
1332
|
-
self.stderr(
|
|
1450
|
+
self.stderr("setulimit", r_id)
|
|
1333
1451
|
u_soft, hard = self.getrlimit(r_id) # @UnusedVariable
|
|
1334
1452
|
if hard != self._rl.RLIM_INFINITY:
|
|
1335
|
-
logger.info(
|
|
1336
|
-
|
|
1453
|
+
logger.info(
|
|
1454
|
+
'Unable to raise the %s soft limit to "unlimited", '
|
|
1455
|
+
+ "using the hard limit instead (%s).",
|
|
1456
|
+
str(r_id),
|
|
1457
|
+
str(hard),
|
|
1458
|
+
)
|
|
1337
1459
|
return self.setrlimit(r_id, (hard, hard))
|
|
1338
1460
|
|
|
1339
1461
|
def ulimit(self):
|
|
1340
1462
|
"""Dump the user limits currently defined."""
|
|
1341
|
-
for limit in [r for r in dir(self._rl) if r.startswith(
|
|
1342
|
-
print(
|
|
1463
|
+
for limit in [r for r in dir(self._rl) if r.startswith("RLIMIT_")]:
|
|
1464
|
+
print(
|
|
1465
|
+
" ",
|
|
1466
|
+
limit.ljust(16),
|
|
1467
|
+
":",
|
|
1468
|
+
self._rl.getrlimit(getattr(self._rl, limit)),
|
|
1469
|
+
)
|
|
1343
1470
|
|
|
1344
1471
|
@property
|
|
1345
1472
|
def cpus_info(self):
|
|
@@ -1352,7 +1479,9 @@ class OSExtended(System):
|
|
|
1352
1479
|
"""
|
|
1353
1480
|
return self._cpusinfo
|
|
1354
1481
|
|
|
1355
|
-
def cpus_ids_per_blocks(
|
|
1482
|
+
def cpus_ids_per_blocks(
|
|
1483
|
+
self, blocksize=1, topology="raw", hexmask=False
|
|
1484
|
+
): # @UnusedVariable
|
|
1356
1485
|
"""Get the list of CPUs IDs ordered for subsequent binding.
|
|
1357
1486
|
|
|
1358
1487
|
:param int blocksize: The number of thread consumed by one task
|
|
@@ -1361,14 +1490,16 @@ class OSExtended(System):
|
|
|
1361
1490
|
"""
|
|
1362
1491
|
return []
|
|
1363
1492
|
|
|
1364
|
-
def cpus_ids_dispenser(self, topology=
|
|
1493
|
+
def cpus_ids_dispenser(self, topology="raw"):
|
|
1365
1494
|
"""Get a dispenser of CPUs IDs for nicely ordered for subsequent binding.
|
|
1366
1495
|
|
|
1367
1496
|
:param str topology: The task distribution scheme
|
|
1368
1497
|
"""
|
|
1369
1498
|
return None
|
|
1370
1499
|
|
|
1371
|
-
def cpus_affinity_get(
|
|
1500
|
+
def cpus_affinity_get(
|
|
1501
|
+
self, taskid, blocksize=1, method="default", topology="raw"
|
|
1502
|
+
): # @UnusedVariable
|
|
1372
1503
|
"""Get the necessary command/environment to set the CPUs affinity.
|
|
1373
1504
|
|
|
1374
1505
|
:param int taskid: the task number
|
|
@@ -1421,7 +1552,9 @@ class OSExtended(System):
|
|
|
1421
1552
|
netstat may not be implemented.
|
|
1422
1553
|
"""
|
|
1423
1554
|
if self.netstatsinfo is None:
|
|
1424
|
-
raise NotImplementedError(
|
|
1555
|
+
raise NotImplementedError(
|
|
1556
|
+
"This function is not implemented on this system."
|
|
1557
|
+
)
|
|
1425
1558
|
return self.netstatsinfo.available_localport()
|
|
1426
1559
|
|
|
1427
1560
|
def check_localport(self, port):
|
|
@@ -1431,12 +1564,14 @@ class OSExtended(System):
|
|
|
1431
1564
|
netstat may not be implemented.
|
|
1432
1565
|
"""
|
|
1433
1566
|
if self.netstatsinfo is None:
|
|
1434
|
-
raise NotImplementedError(
|
|
1567
|
+
raise NotImplementedError(
|
|
1568
|
+
"This function is not implemented on this system."
|
|
1569
|
+
)
|
|
1435
1570
|
return self.netstatsinfo.check_localport(port)
|
|
1436
1571
|
|
|
1437
1572
|
def clear(self):
|
|
1438
1573
|
"""Clone of the unix eponymous command."""
|
|
1439
|
-
self._os.system(
|
|
1574
|
+
self._os.system("clear")
|
|
1440
1575
|
|
|
1441
1576
|
@property
|
|
1442
1577
|
def cls(self):
|
|
@@ -1444,8 +1579,14 @@ class OSExtended(System):
|
|
|
1444
1579
|
self.clear()
|
|
1445
1580
|
return None
|
|
1446
1581
|
|
|
1447
|
-
def rawopts(
|
|
1448
|
-
|
|
1582
|
+
def rawopts(
|
|
1583
|
+
self,
|
|
1584
|
+
cmdline=None,
|
|
1585
|
+
defaults=None,
|
|
1586
|
+
isnone=isnonedef,
|
|
1587
|
+
istrue=istruedef,
|
|
1588
|
+
isfalse=isfalsedef,
|
|
1589
|
+
):
|
|
1449
1590
|
"""Parse a simple options command line that looks like `` key=value``.
|
|
1450
1591
|
|
|
1451
1592
|
:param str cmdline: The command line to be processed (if *None*, ``sys.argv``
|
|
@@ -1461,11 +1602,13 @@ class OSExtended(System):
|
|
|
1461
1602
|
try:
|
|
1462
1603
|
opts.update(defaults)
|
|
1463
1604
|
except (ValueError, TypeError):
|
|
1464
|
-
logger.warning(
|
|
1605
|
+
logger.warning(
|
|
1606
|
+
"Could not update options default: %s", defaults
|
|
1607
|
+
)
|
|
1465
1608
|
|
|
1466
1609
|
if cmdline is None:
|
|
1467
1610
|
cmdline = sys.argv[1:]
|
|
1468
|
-
opts.update(dict([x.split(
|
|
1611
|
+
opts.update(dict([x.split("=") for x in cmdline]))
|
|
1469
1612
|
for k, v in opts.items():
|
|
1470
1613
|
if v not in (None, True, False):
|
|
1471
1614
|
if istrue.match(v):
|
|
@@ -1479,9 +1622,10 @@ class OSExtended(System):
|
|
|
1479
1622
|
def is_iofile(self, iocandidate):
|
|
1480
1623
|
"""Check if actual **iocandidate** is a valid filename or io stream."""
|
|
1481
1624
|
return iocandidate is not None and (
|
|
1482
|
-
(isinstance(iocandidate, str) and self.path.exists(iocandidate))
|
|
1483
|
-
isinstance(iocandidate, io.IOBase)
|
|
1484
|
-
|
|
1625
|
+
(isinstance(iocandidate, str) and self.path.exists(iocandidate))
|
|
1626
|
+
or isinstance(iocandidate, io.IOBase)
|
|
1627
|
+
or isinstance(iocandidate, io.BytesIO)
|
|
1628
|
+
or isinstance(iocandidate, io.StringIO)
|
|
1485
1629
|
)
|
|
1486
1630
|
|
|
1487
1631
|
@contextlib.contextmanager
|
|
@@ -1512,20 +1656,28 @@ class OSExtended(System):
|
|
|
1512
1656
|
hostname = self.glove.default_fthost
|
|
1513
1657
|
if not hostname:
|
|
1514
1658
|
if fatal:
|
|
1515
|
-
raise ValueError(
|
|
1659
|
+
raise ValueError(
|
|
1660
|
+
"An *hostname* must be provided one way or another"
|
|
1661
|
+
)
|
|
1516
1662
|
return hostname
|
|
1517
1663
|
|
|
1518
1664
|
def fix_ftuser(self, hostname, logname, fatal=True, defaults_to_user=True):
|
|
1519
1665
|
"""Given *hostname*, if *logname* is None, tries to find a default value for it."""
|
|
1520
1666
|
if logname is None:
|
|
1521
1667
|
if self.glove is not None:
|
|
1522
|
-
logname = self.glove.getftuser(
|
|
1668
|
+
logname = self.glove.getftuser(
|
|
1669
|
+
hostname, defaults_to_user=defaults_to_user
|
|
1670
|
+
)
|
|
1523
1671
|
else:
|
|
1524
1672
|
if fatal:
|
|
1525
|
-
raise ValueError(
|
|
1673
|
+
raise ValueError(
|
|
1674
|
+
"Either a *logname* or a glove must be set-up"
|
|
1675
|
+
)
|
|
1526
1676
|
return logname
|
|
1527
1677
|
|
|
1528
|
-
def ftp(
|
|
1678
|
+
def ftp(
|
|
1679
|
+
self, hostname, logname=None, delayed=False, port=DEFAULT_FTP_PORT
|
|
1680
|
+
):
|
|
1529
1681
|
"""Return an FTP client object.
|
|
1530
1682
|
|
|
1531
1683
|
:param str hostname: the remote host's name for FTP.
|
|
@@ -1555,20 +1707,36 @@ class OSExtended(System):
|
|
|
1555
1707
|
logname = self.fix_ftuser(hostname, logname)
|
|
1556
1708
|
if port is None:
|
|
1557
1709
|
port = DEFAULT_FTP_PORT
|
|
1558
|
-
if
|
|
1559
|
-
|
|
1710
|
+
if (
|
|
1711
|
+
self.ftpflavour == FTP_FLAVOUR.CONNECTION_POOLS
|
|
1712
|
+
and self._current_ftppool is not None
|
|
1713
|
+
):
|
|
1714
|
+
return self._current_ftppool.deal(
|
|
1715
|
+
hostname, logname, port=port, delayed=delayed
|
|
1716
|
+
)
|
|
1560
1717
|
else:
|
|
1561
|
-
ftpclass =
|
|
1718
|
+
ftpclass = (
|
|
1719
|
+
AutoRetriesFtp
|
|
1720
|
+
if self.ftpflavour != FTP_FLAVOUR.STD
|
|
1721
|
+
else StdFtp
|
|
1722
|
+
)
|
|
1562
1723
|
ftpbox = ftpclass(self, hostname, port=port)
|
|
1563
1724
|
rc = ftpbox.fastlogin(logname, delayed=delayed)
|
|
1564
1725
|
if rc:
|
|
1565
1726
|
return ftpbox
|
|
1566
1727
|
else:
|
|
1567
|
-
logger.warning(
|
|
1728
|
+
logger.warning(
|
|
1729
|
+
"Could not login on %s as %s [%s]",
|
|
1730
|
+
hostname,
|
|
1731
|
+
logname,
|
|
1732
|
+
str(rc),
|
|
1733
|
+
)
|
|
1568
1734
|
return None
|
|
1569
1735
|
|
|
1570
1736
|
@contextlib.contextmanager
|
|
1571
|
-
def ftpcontext(
|
|
1737
|
+
def ftpcontext(
|
|
1738
|
+
self, hostname, logname=None, delayed=False, port=DEFAULT_FTP_PORT
|
|
1739
|
+
):
|
|
1572
1740
|
"""Create an FTP object and close it when the context exits.
|
|
1573
1741
|
|
|
1574
1742
|
For a description of the context's arguments, see :func:`ftp`.
|
|
@@ -1591,8 +1759,15 @@ class OSExtended(System):
|
|
|
1591
1759
|
return remote
|
|
1592
1760
|
|
|
1593
1761
|
@fmtshcmd
|
|
1594
|
-
def ftget(
|
|
1595
|
-
|
|
1762
|
+
def ftget(
|
|
1763
|
+
self,
|
|
1764
|
+
source,
|
|
1765
|
+
destination,
|
|
1766
|
+
hostname=None,
|
|
1767
|
+
logname=None,
|
|
1768
|
+
port=DEFAULT_FTP_PORT,
|
|
1769
|
+
cpipeline=None,
|
|
1770
|
+
):
|
|
1596
1771
|
"""Proceed to a direct ftp get on the specified target (using Vortex's FTP client).
|
|
1597
1772
|
|
|
1598
1773
|
:param str source: the remote path to get data
|
|
@@ -1616,10 +1791,12 @@ class OSExtended(System):
|
|
|
1616
1791
|
if cpipeline is None:
|
|
1617
1792
|
rc = ftp.get(source, destination)
|
|
1618
1793
|
else:
|
|
1619
|
-
with cpipeline.stream2uncompress(
|
|
1794
|
+
with cpipeline.stream2uncompress(
|
|
1795
|
+
destination
|
|
1796
|
+
) as cdestination:
|
|
1620
1797
|
rc = ftp.get(source, cdestination)
|
|
1621
1798
|
except ftplib.all_errors as e:
|
|
1622
|
-
logger.warning(
|
|
1799
|
+
logger.warning("An FTP error occured: %s", str(e))
|
|
1623
1800
|
rc = False
|
|
1624
1801
|
finally:
|
|
1625
1802
|
ftp.close()
|
|
@@ -1628,8 +1805,16 @@ class OSExtended(System):
|
|
|
1628
1805
|
return False
|
|
1629
1806
|
|
|
1630
1807
|
@fmtshcmd
|
|
1631
|
-
def ftput(
|
|
1632
|
-
|
|
1808
|
+
def ftput(
|
|
1809
|
+
self,
|
|
1810
|
+
source,
|
|
1811
|
+
destination,
|
|
1812
|
+
hostname=None,
|
|
1813
|
+
logname=None,
|
|
1814
|
+
port=DEFAULT_FTP_PORT,
|
|
1815
|
+
cpipeline=None,
|
|
1816
|
+
sync=False,
|
|
1817
|
+
): # @UnusedVariable
|
|
1633
1818
|
"""Proceed to a direct ftp put on the specified target (using Vortex's FTP client).
|
|
1634
1819
|
|
|
1635
1820
|
:param source: The source of data (either a path to file or a
|
|
@@ -1655,33 +1840,45 @@ class OSExtended(System):
|
|
|
1655
1840
|
if cpipeline is None:
|
|
1656
1841
|
rc = ftp.put(source, destination)
|
|
1657
1842
|
else:
|
|
1658
|
-
with cpipeline.compress2stream(
|
|
1843
|
+
with cpipeline.compress2stream(
|
|
1844
|
+
source, iosponge=True
|
|
1845
|
+
) as csource:
|
|
1659
1846
|
# csource is an IoSponge consequently the size attribute exists
|
|
1660
|
-
rc = ftp.put(
|
|
1847
|
+
rc = ftp.put(
|
|
1848
|
+
csource, destination, size=csource.size
|
|
1849
|
+
)
|
|
1661
1850
|
except ftplib.all_errors as e:
|
|
1662
|
-
logger.warning(
|
|
1851
|
+
logger.warning("An FTP error occured: %s", str(e))
|
|
1663
1852
|
rc = False
|
|
1664
1853
|
finally:
|
|
1665
1854
|
ftp.close()
|
|
1666
1855
|
else:
|
|
1667
|
-
raise OSError(
|
|
1856
|
+
raise OSError("No such file or directory: {!r}".format(source))
|
|
1668
1857
|
return rc
|
|
1669
1858
|
|
|
1670
1859
|
def ftspool_cache(self):
|
|
1671
1860
|
"""Return a cache object for the FtSpool."""
|
|
1672
|
-
if self._ftspool_cache is None:
|
|
1673
|
-
self._ftspool_cache
|
|
1861
|
+
if self._ftspool_cache is not None:
|
|
1862
|
+
return self._ftspool_cache
|
|
1863
|
+
self._ftspool_cache = footprints.proxy.cache(
|
|
1864
|
+
entry=os.path.join(
|
|
1865
|
+
vortex.data.stores.get_cache_location(), "ftspool"
|
|
1866
|
+
),
|
|
1867
|
+
)
|
|
1674
1868
|
return self._ftspool_cache
|
|
1675
1869
|
|
|
1676
1870
|
def copy2ftspool(self, source, nest=False, **kwargs):
|
|
1677
1871
|
"""Make a copy of **source** to the FtSpool cache."""
|
|
1678
|
-
h = hashlib.new(
|
|
1679
|
-
h.update(source.encode(encoding=
|
|
1680
|
-
outputname =
|
|
1681
|
-
|
|
1872
|
+
h = hashlib.new("md5")
|
|
1873
|
+
h.update(source.encode(encoding="utf-8"))
|
|
1874
|
+
outputname = "vortex_{:s}_P{:06d}_{:s}".format(
|
|
1875
|
+
date.now().strftime("%Y%m%d%H%M%S-%f"),
|
|
1876
|
+
self.getpid(),
|
|
1877
|
+
h.hexdigest(),
|
|
1878
|
+
)
|
|
1682
1879
|
if nest:
|
|
1683
1880
|
outputname = self.path.join(outputname, self.path.basename(source))
|
|
1684
|
-
kwargs[
|
|
1881
|
+
kwargs["intent"] = "in" # Force intent=in
|
|
1685
1882
|
if self.ftspool_cache().insert(outputname, source, **kwargs):
|
|
1686
1883
|
return self.ftspool_cache().fullpath(outputname)
|
|
1687
1884
|
else:
|
|
@@ -1691,42 +1888,69 @@ class OSExtended(System):
|
|
|
1691
1888
|
"""Given **source** and **destination**, is FtServ usable ?"""
|
|
1692
1889
|
return isinstance(source, str) and isinstance(destination, str)
|
|
1693
1890
|
|
|
1694
|
-
def ftserv_put(
|
|
1695
|
-
|
|
1891
|
+
def ftserv_put(
|
|
1892
|
+
self,
|
|
1893
|
+
source,
|
|
1894
|
+
destination,
|
|
1895
|
+
hostname=None,
|
|
1896
|
+
logname=None,
|
|
1897
|
+
port=None,
|
|
1898
|
+
specialshell=None,
|
|
1899
|
+
sync=False,
|
|
1900
|
+
):
|
|
1696
1901
|
"""Asynchronous put of a file using FtServ."""
|
|
1697
1902
|
if self.ftserv_allowed(source, destination):
|
|
1698
1903
|
if self.path.exists(source):
|
|
1699
|
-
ftcmd = self.ftputcmd or
|
|
1904
|
+
ftcmd = self.ftputcmd or "ftput"
|
|
1700
1905
|
hostname = self.fix_fthostname(hostname, fatal=False)
|
|
1701
1906
|
logname = self.fix_ftuser(hostname, logname, fatal=False)
|
|
1702
1907
|
extras = list()
|
|
1703
1908
|
if not sync:
|
|
1704
|
-
extras.extend(
|
|
1909
|
+
extras.extend(
|
|
1910
|
+
[
|
|
1911
|
+
"-q",
|
|
1912
|
+
]
|
|
1913
|
+
)
|
|
1705
1914
|
if hostname:
|
|
1706
1915
|
if port is not None:
|
|
1707
|
-
hostname +=
|
|
1708
|
-
extras.extend([
|
|
1916
|
+
hostname += ":{:s}".format(port)
|
|
1917
|
+
extras.extend(["-h", hostname])
|
|
1709
1918
|
if logname:
|
|
1710
|
-
extras.extend([
|
|
1919
|
+
extras.extend(["-u", logname])
|
|
1711
1920
|
if specialshell:
|
|
1712
|
-
extras.extend([
|
|
1921
|
+
extras.extend(["-s", specialshell])
|
|
1713
1922
|
# Remove ~/ and ~logname/ from the destinations' path
|
|
1714
|
-
actual_dest = re.sub(
|
|
1923
|
+
actual_dest = re.sub("^~/+", "", destination)
|
|
1715
1924
|
if logname:
|
|
1716
|
-
actual_dest = re.sub(
|
|
1925
|
+
actual_dest = re.sub(
|
|
1926
|
+
"^~{:s}/+".format(logname), "", actual_dest
|
|
1927
|
+
)
|
|
1717
1928
|
try:
|
|
1718
|
-
rc = self.spawn(
|
|
1719
|
-
|
|
1720
|
-
|
|
1929
|
+
rc = self.spawn(
|
|
1930
|
+
[
|
|
1931
|
+
ftcmd,
|
|
1932
|
+
"-o",
|
|
1933
|
+
"mkdir",
|
|
1934
|
+
] # Automatically create subdirectories
|
|
1935
|
+
+ extras
|
|
1936
|
+
+ [source, actual_dest],
|
|
1937
|
+
output=False,
|
|
1938
|
+
)
|
|
1721
1939
|
except ExecutionError:
|
|
1722
1940
|
rc = False
|
|
1723
1941
|
else:
|
|
1724
|
-
raise OSError(
|
|
1942
|
+
raise OSError("No such file or directory: {!s}".format(source))
|
|
1725
1943
|
else:
|
|
1726
|
-
raise OSError(
|
|
1944
|
+
raise OSError(
|
|
1945
|
+
"Source or destination is not a plain file path: {!r}".format(
|
|
1946
|
+
source
|
|
1947
|
+
)
|
|
1948
|
+
)
|
|
1727
1949
|
return rc
|
|
1728
1950
|
|
|
1729
|
-
def ftserv_get(
|
|
1951
|
+
def ftserv_get(
|
|
1952
|
+
self, source, destination, hostname=None, logname=None, port=None
|
|
1953
|
+
):
|
|
1730
1954
|
"""Get a file using FtServ."""
|
|
1731
1955
|
if self.ftserv_allowed(source, destination):
|
|
1732
1956
|
if self.filecocoon(destination):
|
|
@@ -1736,67 +1960,103 @@ class OSExtended(System):
|
|
|
1736
1960
|
extras = list()
|
|
1737
1961
|
if hostname:
|
|
1738
1962
|
if port is not None:
|
|
1739
|
-
hostname +=
|
|
1740
|
-
extras.extend([
|
|
1963
|
+
hostname += ":{:s}".format(port)
|
|
1964
|
+
extras.extend(["-h", hostname])
|
|
1741
1965
|
if logname:
|
|
1742
|
-
extras.extend([
|
|
1743
|
-
ftcmd = self.ftgetcmd or
|
|
1966
|
+
extras.extend(["-u", logname])
|
|
1967
|
+
ftcmd = self.ftgetcmd or "ftget"
|
|
1744
1968
|
try:
|
|
1745
|
-
rc = self.spawn(
|
|
1969
|
+
rc = self.spawn(
|
|
1970
|
+
[
|
|
1971
|
+
ftcmd,
|
|
1972
|
+
]
|
|
1973
|
+
+ extras
|
|
1974
|
+
+ [source, destination],
|
|
1975
|
+
output=False,
|
|
1976
|
+
)
|
|
1746
1977
|
except ExecutionError:
|
|
1747
1978
|
rc = False
|
|
1748
1979
|
else:
|
|
1749
|
-
raise OSError(
|
|
1980
|
+
raise OSError("Could not cocoon: {!s}".format(destination))
|
|
1750
1981
|
else:
|
|
1751
|
-
raise OSError(
|
|
1982
|
+
raise OSError(
|
|
1983
|
+
"Source or destination is not a plain file path: {!r}".format(
|
|
1984
|
+
source
|
|
1985
|
+
)
|
|
1986
|
+
)
|
|
1752
1987
|
return rc
|
|
1753
1988
|
|
|
1754
|
-
def ftserv_batchget(
|
|
1989
|
+
def ftserv_batchget(
|
|
1990
|
+
self, source, destination, hostname=None, logname=None, port=None
|
|
1991
|
+
):
|
|
1755
1992
|
"""Get a list of files using FtServ.
|
|
1756
1993
|
|
|
1757
1994
|
:note: **source** and **destination** are list or tuple.
|
|
1758
1995
|
"""
|
|
1759
|
-
if all(
|
|
1996
|
+
if all(
|
|
1997
|
+
[self.ftserv_allowed(s, d) for s, d in zip(source, destination)]
|
|
1998
|
+
):
|
|
1760
1999
|
for d in destination:
|
|
1761
2000
|
if not self.filecocoon(d):
|
|
1762
|
-
raise OSError(
|
|
2001
|
+
raise OSError("Could not cocoon: {!s}".format(d))
|
|
1763
2002
|
extras = list()
|
|
1764
2003
|
hostname = self.fix_fthostname(hostname, fatal=False)
|
|
1765
2004
|
logname = self.fix_ftuser(hostname, logname, fatal=False)
|
|
1766
2005
|
if hostname:
|
|
1767
2006
|
if port is not None:
|
|
1768
|
-
hostname +=
|
|
1769
|
-
extras.extend([
|
|
2007
|
+
hostname += ":{:s}".format(port)
|
|
2008
|
+
extras.extend(["-h", hostname])
|
|
1770
2009
|
if logname:
|
|
1771
|
-
extras.extend([
|
|
1772
|
-
ftcmd = self.ftgetcmd or
|
|
1773
|
-
plocale = locale.getlocale()[1] or
|
|
1774
|
-
with tempfile.TemporaryFile(
|
|
1775
|
-
|
|
1776
|
-
|
|
1777
|
-
|
|
2010
|
+
extras.extend(["-u", logname])
|
|
2011
|
+
ftcmd = self.ftgetcmd or "ftget"
|
|
2012
|
+
plocale = locale.getlocale()[1] or "ascii"
|
|
2013
|
+
with tempfile.TemporaryFile(
|
|
2014
|
+
dir=self.path.dirname(self.path.abspath(destination[0])),
|
|
2015
|
+
mode="wb",
|
|
2016
|
+
) as tmpio:
|
|
2017
|
+
tmpio.writelines(
|
|
2018
|
+
[
|
|
2019
|
+
"{:s} {:s}\n".format(s, d).encode(plocale)
|
|
2020
|
+
for s, d in zip(source, destination)
|
|
2021
|
+
]
|
|
2022
|
+
)
|
|
1778
2023
|
tmpio.seek(0)
|
|
1779
|
-
with tempfile.TemporaryFile(
|
|
1780
|
-
|
|
2024
|
+
with tempfile.TemporaryFile(
|
|
2025
|
+
dir=self.path.dirname(self.path.abspath(destination[0])),
|
|
2026
|
+
mode="w+b",
|
|
2027
|
+
) as tmpoutput:
|
|
1781
2028
|
try:
|
|
1782
|
-
rc = self.spawn(
|
|
2029
|
+
rc = self.spawn(
|
|
2030
|
+
[
|
|
2031
|
+
ftcmd,
|
|
2032
|
+
]
|
|
2033
|
+
+ extras,
|
|
2034
|
+
output=tmpoutput,
|
|
2035
|
+
stdin=tmpio,
|
|
2036
|
+
)
|
|
1783
2037
|
except ExecutionError:
|
|
1784
2038
|
rc = False
|
|
1785
2039
|
# Process output data
|
|
1786
2040
|
tmpoutput.seek(0)
|
|
1787
2041
|
ft_outputs = tmpoutput.read()
|
|
1788
|
-
ft_outputs = ft_outputs.decode(
|
|
1789
|
-
|
|
2042
|
+
ft_outputs = ft_outputs.decode(
|
|
2043
|
+
locale.getlocale()[1] or "ascii", "replace"
|
|
2044
|
+
)
|
|
2045
|
+
logger.info("Here is the ftget command output: \n%s", ft_outputs)
|
|
1790
2046
|
# Expand the return codes
|
|
1791
2047
|
if rc:
|
|
1792
|
-
x_rc = [
|
|
2048
|
+
x_rc = [
|
|
2049
|
+
True,
|
|
2050
|
+
] * len(source)
|
|
1793
2051
|
else:
|
|
1794
|
-
ack_re = re.compile(r
|
|
2052
|
+
ack_re = re.compile(r".*FT_(OK|ABORT)\s*:\s*GET\s+(.*)$")
|
|
1795
2053
|
ack_lines = dict()
|
|
1796
|
-
for line in ft_outputs.split(
|
|
2054
|
+
for line in ft_outputs.split("\n"):
|
|
1797
2055
|
ack_match = ack_re.match(line)
|
|
1798
2056
|
if ack_match:
|
|
1799
|
-
ack_lines[ack_match.group(2)] =
|
|
2057
|
+
ack_lines[ack_match.group(2)] = (
|
|
2058
|
+
ack_match.group(1) == "OK"
|
|
2059
|
+
)
|
|
1800
2060
|
x_rc = []
|
|
1801
2061
|
for a_source in source:
|
|
1802
2062
|
my_rc = None
|
|
@@ -1806,16 +2066,28 @@ class OSExtended(System):
|
|
|
1806
2066
|
break
|
|
1807
2067
|
x_rc.append(my_rc)
|
|
1808
2068
|
else:
|
|
1809
|
-
raise OSError(
|
|
2069
|
+
raise OSError(
|
|
2070
|
+
"Source or destination is not a plain file path: {!r}".format(
|
|
2071
|
+
source
|
|
2072
|
+
)
|
|
2073
|
+
)
|
|
1810
2074
|
return x_rc
|
|
1811
2075
|
|
|
1812
2076
|
def rawftput_worthy(self, source, destination):
|
|
1813
2077
|
"""Is it allowed to use FtServ given **source** and **destination**."""
|
|
1814
|
-
return self.
|
|
2078
|
+
return self.ftserv and self.ftserv_allowed(source, destination)
|
|
1815
2079
|
|
|
1816
2080
|
@fmtshcmd
|
|
1817
|
-
def rawftput(
|
|
1818
|
-
|
|
2081
|
+
def rawftput(
|
|
2082
|
+
self,
|
|
2083
|
+
source,
|
|
2084
|
+
destination,
|
|
2085
|
+
hostname=None,
|
|
2086
|
+
logname=None,
|
|
2087
|
+
port=None,
|
|
2088
|
+
cpipeline=None,
|
|
2089
|
+
sync=False,
|
|
2090
|
+
):
|
|
1819
2091
|
"""Proceed with some external ftput command on the specified target.
|
|
1820
2092
|
|
|
1821
2093
|
:param str source: Path to the source filename
|
|
@@ -1829,21 +2101,43 @@ class OSExtended(System):
|
|
|
1829
2101
|
"""
|
|
1830
2102
|
if cpipeline is not None:
|
|
1831
2103
|
if cpipeline.compress2rawftp(source):
|
|
1832
|
-
return self.ftserv_put(
|
|
1833
|
-
|
|
1834
|
-
|
|
1835
|
-
|
|
2104
|
+
return self.ftserv_put(
|
|
2105
|
+
source,
|
|
2106
|
+
destination,
|
|
2107
|
+
hostname,
|
|
2108
|
+
logname=logname,
|
|
2109
|
+
port=port,
|
|
2110
|
+
specialshell=cpipeline.compress2rawftp(source),
|
|
2111
|
+
sync=sync,
|
|
2112
|
+
)
|
|
1836
2113
|
else:
|
|
1837
2114
|
if port is None:
|
|
1838
2115
|
port = DEFAULT_FTP_PORT
|
|
1839
|
-
return self.ftput(
|
|
1840
|
-
|
|
2116
|
+
return self.ftput(
|
|
2117
|
+
source,
|
|
2118
|
+
destination,
|
|
2119
|
+
hostname,
|
|
2120
|
+
logname=logname,
|
|
2121
|
+
port=port,
|
|
2122
|
+
cpipeline=cpipeline,
|
|
2123
|
+
sync=sync,
|
|
2124
|
+
)
|
|
1841
2125
|
else:
|
|
1842
|
-
return self.ftserv_put(
|
|
1843
|
-
|
|
2126
|
+
return self.ftserv_put(
|
|
2127
|
+
source, destination, hostname, logname, port=port, sync=sync
|
|
2128
|
+
)
|
|
1844
2129
|
|
|
1845
|
-
def smartftput(
|
|
1846
|
-
|
|
2130
|
+
def smartftput(
|
|
2131
|
+
self,
|
|
2132
|
+
source,
|
|
2133
|
+
destination,
|
|
2134
|
+
hostname=None,
|
|
2135
|
+
logname=None,
|
|
2136
|
+
port=None,
|
|
2137
|
+
cpipeline=None,
|
|
2138
|
+
sync=False,
|
|
2139
|
+
fmt=None,
|
|
2140
|
+
):
|
|
1847
2141
|
"""Select the best alternative between ``ftput`` and ``rawftput``.
|
|
1848
2142
|
|
|
1849
2143
|
:param source: The source of data (either a path to file or a
|
|
@@ -1862,26 +2156,53 @@ class OSExtended(System):
|
|
|
1862
2156
|
|
|
1863
2157
|
``rawftput`` will be used if all of the following conditions are met:
|
|
1864
2158
|
|
|
1865
|
-
* ``self.
|
|
2159
|
+
* ``self.ftserv`` is *True*
|
|
1866
2160
|
* **source** is a string (as opposed to a File like object)
|
|
1867
2161
|
* **destination** is a string (as opposed to a File like object)
|
|
1868
2162
|
"""
|
|
1869
2163
|
if self.rawftput_worthy(source, destination):
|
|
1870
|
-
return self.rawftput(
|
|
1871
|
-
|
|
2164
|
+
return self.rawftput(
|
|
2165
|
+
source,
|
|
2166
|
+
destination,
|
|
2167
|
+
hostname=hostname,
|
|
2168
|
+
logname=logname,
|
|
2169
|
+
port=port,
|
|
2170
|
+
cpipeline=cpipeline,
|
|
2171
|
+
sync=sync,
|
|
2172
|
+
fmt=fmt,
|
|
2173
|
+
)
|
|
1872
2174
|
else:
|
|
1873
2175
|
if port is None:
|
|
1874
2176
|
port = DEFAULT_FTP_PORT
|
|
1875
|
-
return self.ftput(
|
|
1876
|
-
|
|
2177
|
+
return self.ftput(
|
|
2178
|
+
source,
|
|
2179
|
+
destination,
|
|
2180
|
+
hostname=hostname,
|
|
2181
|
+
logname=logname,
|
|
2182
|
+
port=port,
|
|
2183
|
+
cpipeline=cpipeline,
|
|
2184
|
+
sync=sync,
|
|
2185
|
+
fmt=fmt,
|
|
2186
|
+
)
|
|
1877
2187
|
|
|
1878
2188
|
def rawftget_worthy(self, source, destination, cpipeline=None):
|
|
1879
2189
|
"""Is it allowed to use FtServ given **source** and **destination**."""
|
|
1880
|
-
return
|
|
2190
|
+
return (
|
|
2191
|
+
self.ftserv
|
|
2192
|
+
and cpipeline is None
|
|
2193
|
+
and self.ftserv_allowed(source, destination)
|
|
2194
|
+
)
|
|
1881
2195
|
|
|
1882
2196
|
@fmtshcmd
|
|
1883
|
-
def rawftget(
|
|
1884
|
-
|
|
2197
|
+
def rawftget(
|
|
2198
|
+
self,
|
|
2199
|
+
source,
|
|
2200
|
+
destination,
|
|
2201
|
+
hostname=None,
|
|
2202
|
+
logname=None,
|
|
2203
|
+
port=None,
|
|
2204
|
+
cpipeline=None,
|
|
2205
|
+
):
|
|
1885
2206
|
"""Proceed with some external ftget command on the specified target.
|
|
1886
2207
|
|
|
1887
2208
|
:param str source: the remote path to get data
|
|
@@ -1893,11 +2214,20 @@ class OSExtended(System):
|
|
|
1893
2214
|
"""
|
|
1894
2215
|
if cpipeline is not None:
|
|
1895
2216
|
raise OSError("cpipeline is not supported by this method.")
|
|
1896
|
-
return self.ftserv_get(
|
|
2217
|
+
return self.ftserv_get(
|
|
2218
|
+
source, destination, hostname, logname, port=port
|
|
2219
|
+
)
|
|
1897
2220
|
|
|
1898
2221
|
@fmtshcmd
|
|
1899
|
-
def batchrawftget(
|
|
1900
|
-
|
|
2222
|
+
def batchrawftget(
|
|
2223
|
+
self,
|
|
2224
|
+
source,
|
|
2225
|
+
destination,
|
|
2226
|
+
hostname=None,
|
|
2227
|
+
logname=None,
|
|
2228
|
+
port=None,
|
|
2229
|
+
cpipeline=None,
|
|
2230
|
+
):
|
|
1901
2231
|
"""Proceed with some external ftget command on the specified target.
|
|
1902
2232
|
|
|
1903
2233
|
:param source: A list of remote paths to get data
|
|
@@ -1909,10 +2239,20 @@ class OSExtended(System):
|
|
|
1909
2239
|
"""
|
|
1910
2240
|
if cpipeline is not None:
|
|
1911
2241
|
raise OSError("cpipeline is not supported by this method.")
|
|
1912
|
-
return self.ftserv_batchget(
|
|
2242
|
+
return self.ftserv_batchget(
|
|
2243
|
+
source, destination, hostname, logname, port=port
|
|
2244
|
+
)
|
|
1913
2245
|
|
|
1914
|
-
def smartftget(
|
|
1915
|
-
|
|
2246
|
+
def smartftget(
|
|
2247
|
+
self,
|
|
2248
|
+
source,
|
|
2249
|
+
destination,
|
|
2250
|
+
hostname=None,
|
|
2251
|
+
logname=None,
|
|
2252
|
+
port=None,
|
|
2253
|
+
cpipeline=None,
|
|
2254
|
+
fmt=None,
|
|
2255
|
+
):
|
|
1916
2256
|
"""Select the best alternative between ``ftget`` and ``rawftget``.
|
|
1917
2257
|
|
|
1918
2258
|
:param str source: the remote path to get data
|
|
@@ -1930,23 +2270,45 @@ class OSExtended(System):
|
|
|
1930
2270
|
|
|
1931
2271
|
``rawftget`` will be used if all of the following conditions are met:
|
|
1932
2272
|
|
|
1933
|
-
* ``self.
|
|
2273
|
+
* ``self.ftserv`` is *True*
|
|
1934
2274
|
* **cpipeline** is None
|
|
1935
2275
|
* **source** is a string (as opposed to a File like object)
|
|
1936
2276
|
* **destination** is a string (as opposed to a File like object)
|
|
1937
2277
|
"""
|
|
1938
2278
|
if self.rawftget_worthy(source, destination, cpipeline):
|
|
1939
2279
|
# FtServ is uninteresting when dealing with compression
|
|
1940
|
-
return self.rawftget(
|
|
1941
|
-
|
|
2280
|
+
return self.rawftget(
|
|
2281
|
+
source,
|
|
2282
|
+
destination,
|
|
2283
|
+
hostname=hostname,
|
|
2284
|
+
logname=logname,
|
|
2285
|
+
port=port,
|
|
2286
|
+
cpipeline=cpipeline,
|
|
2287
|
+
fmt=fmt,
|
|
2288
|
+
)
|
|
1942
2289
|
else:
|
|
1943
2290
|
if port is None:
|
|
1944
2291
|
port = DEFAULT_FTP_PORT
|
|
1945
|
-
return self.ftget(
|
|
1946
|
-
|
|
2292
|
+
return self.ftget(
|
|
2293
|
+
source,
|
|
2294
|
+
destination,
|
|
2295
|
+
hostname=hostname,
|
|
2296
|
+
logname=logname,
|
|
2297
|
+
port=port,
|
|
2298
|
+
cpipeline=cpipeline,
|
|
2299
|
+
fmt=fmt,
|
|
2300
|
+
)
|
|
1947
2301
|
|
|
1948
|
-
def smartbatchftget(
|
|
1949
|
-
|
|
2302
|
+
def smartbatchftget(
|
|
2303
|
+
self,
|
|
2304
|
+
source,
|
|
2305
|
+
destination,
|
|
2306
|
+
hostname=None,
|
|
2307
|
+
logname=None,
|
|
2308
|
+
port=None,
|
|
2309
|
+
cpipeline=None,
|
|
2310
|
+
fmt=None,
|
|
2311
|
+
):
|
|
1950
2312
|
"""
|
|
1951
2313
|
Select the best alternative between ``ftget`` and ``batchrawftget``
|
|
1952
2314
|
when retrieving several files.
|
|
@@ -1964,18 +2326,37 @@ class OSExtended(System):
|
|
|
1964
2326
|
uncompress the data during the file transfer.
|
|
1965
2327
|
:param str fmt: The format of data.
|
|
1966
2328
|
"""
|
|
1967
|
-
if all(
|
|
2329
|
+
if all(
|
|
2330
|
+
[
|
|
2331
|
+
self.rawftget_worthy(s, d, cpipeline)
|
|
2332
|
+
for s, d in zip(source, destination)
|
|
2333
|
+
]
|
|
2334
|
+
):
|
|
1968
2335
|
# FtServ is uninteresting when dealing with compression
|
|
1969
|
-
return self.batchrawftget(
|
|
1970
|
-
|
|
2336
|
+
return self.batchrawftget(
|
|
2337
|
+
source,
|
|
2338
|
+
destination,
|
|
2339
|
+
hostname=hostname,
|
|
2340
|
+
logname=logname,
|
|
2341
|
+
port=None,
|
|
2342
|
+
cpipeline=cpipeline,
|
|
2343
|
+
fmt=fmt,
|
|
2344
|
+
)
|
|
1971
2345
|
else:
|
|
1972
2346
|
rc = True
|
|
1973
2347
|
if port is None:
|
|
1974
2348
|
port = DEFAULT_FTP_PORT
|
|
1975
2349
|
with self.ftppool():
|
|
1976
2350
|
for s, d in zip(source, destination):
|
|
1977
|
-
rc = rc and self.ftget(
|
|
1978
|
-
|
|
2351
|
+
rc = rc and self.ftget(
|
|
2352
|
+
s,
|
|
2353
|
+
d,
|
|
2354
|
+
hostname=hostname,
|
|
2355
|
+
logname=logname,
|
|
2356
|
+
port=port,
|
|
2357
|
+
cpipeline=cpipeline,
|
|
2358
|
+
fmt=fmt,
|
|
2359
|
+
)
|
|
1979
2360
|
return rc
|
|
1980
2361
|
|
|
1981
2362
|
def ssh(self, hostname, logname=None, *args, **kw):
|
|
@@ -1993,7 +2374,9 @@ class OSExtended(System):
|
|
|
1993
2374
|
return AssistedSsh(self, hostname, logname, *args, **kw)
|
|
1994
2375
|
|
|
1995
2376
|
@fmtshcmd
|
|
1996
|
-
def scpput(
|
|
2377
|
+
def scpput(
|
|
2378
|
+
self, source, destination, hostname, logname=None, cpipeline=None
|
|
2379
|
+
):
|
|
1997
2380
|
"""Perform an scp to the specified target.
|
|
1998
2381
|
|
|
1999
2382
|
:param source: The source of data (either a path to file or a
|
|
@@ -2005,14 +2388,16 @@ class OSExtended(System):
|
|
|
2005
2388
|
:param CompressionPipeline cpipeline: If not *None*, the object used to
|
|
2006
2389
|
compress the data during the file transfer (default: *None*).
|
|
2007
2390
|
"""
|
|
2008
|
-
logname = self.fix_ftuser(
|
|
2009
|
-
|
|
2391
|
+
logname = self.fix_ftuser(
|
|
2392
|
+
hostname, logname, fatal=False, defaults_to_user=False
|
|
2393
|
+
)
|
|
2394
|
+
msg = "[hostname={!s} logname={!s}]".format(hostname, logname)
|
|
2010
2395
|
ssh = self.ssh(hostname, logname)
|
|
2011
2396
|
if isinstance(source, str) and cpipeline is None:
|
|
2012
|
-
self.stderr(
|
|
2397
|
+
self.stderr("scpput", source, destination, msg)
|
|
2013
2398
|
return ssh.scpput(source, destination)
|
|
2014
2399
|
else:
|
|
2015
|
-
self.stderr(
|
|
2400
|
+
self.stderr("scpput_stream", source, destination, msg)
|
|
2016
2401
|
if cpipeline is None:
|
|
2017
2402
|
return ssh.scpput_stream(source, destination)
|
|
2018
2403
|
else:
|
|
@@ -2020,7 +2405,9 @@ class OSExtended(System):
|
|
|
2020
2405
|
return ssh.scpput_stream(csource, destination)
|
|
2021
2406
|
|
|
2022
2407
|
@fmtshcmd
|
|
2023
|
-
def scpget(
|
|
2408
|
+
def scpget(
|
|
2409
|
+
self, source, destination, hostname, logname=None, cpipeline=None
|
|
2410
|
+
):
|
|
2024
2411
|
"""Perform an scp to get the specified source.
|
|
2025
2412
|
|
|
2026
2413
|
:param str source: the remote path to get data
|
|
@@ -2032,14 +2419,16 @@ class OSExtended(System):
|
|
|
2032
2419
|
:param CompressionPipeline cpipeline: If not *None*, the object used to
|
|
2033
2420
|
uncompress the data during the file transfer (default: *None*).
|
|
2034
2421
|
"""
|
|
2035
|
-
logname = self.fix_ftuser(
|
|
2036
|
-
|
|
2422
|
+
logname = self.fix_ftuser(
|
|
2423
|
+
hostname, logname, fatal=False, defaults_to_user=False
|
|
2424
|
+
)
|
|
2425
|
+
msg = "[hostname={!s} logname={!s}]".format(hostname, logname)
|
|
2037
2426
|
ssh = self.ssh(hostname, logname)
|
|
2038
2427
|
if isinstance(destination, str) and cpipeline is None:
|
|
2039
|
-
self.stderr(
|
|
2428
|
+
self.stderr("scpget", source, destination, msg)
|
|
2040
2429
|
return ssh.scpget(source, destination)
|
|
2041
2430
|
else:
|
|
2042
|
-
self.stderr(
|
|
2431
|
+
self.stderr("scpget_stream", source, destination, msg)
|
|
2043
2432
|
if cpipeline is None:
|
|
2044
2433
|
return ssh.scpget_stream(source, destination)
|
|
2045
2434
|
else:
|
|
@@ -2048,7 +2437,7 @@ class OSExtended(System):
|
|
|
2048
2437
|
|
|
2049
2438
|
def softlink(self, source, destination):
|
|
2050
2439
|
"""Set a symbolic link if **source** is not **destination**."""
|
|
2051
|
-
self.stderr(
|
|
2440
|
+
self.stderr("softlink", source, destination)
|
|
2052
2441
|
if source == destination:
|
|
2053
2442
|
return False
|
|
2054
2443
|
else:
|
|
@@ -2057,7 +2446,7 @@ class OSExtended(System):
|
|
|
2057
2446
|
def size(self, filepath):
|
|
2058
2447
|
"""Returns the actual size in bytes of the specified **filepath**."""
|
|
2059
2448
|
filepath = self.path.expanduser(filepath)
|
|
2060
|
-
self.stderr(
|
|
2449
|
+
self.stderr("size", filepath)
|
|
2061
2450
|
try:
|
|
2062
2451
|
return self.stat(filepath).st_size
|
|
2063
2452
|
except Exception:
|
|
@@ -2076,7 +2465,9 @@ class OSExtended(System):
|
|
|
2076
2465
|
total_size = self.size(objpath)
|
|
2077
2466
|
for dirpath, dirnames, filenames in self.walk(objpath):
|
|
2078
2467
|
for f in filenames + dirnames:
|
|
2079
|
-
total_size += self.lstat(
|
|
2468
|
+
total_size += self.lstat(
|
|
2469
|
+
self.path.join(dirpath, f)
|
|
2470
|
+
).st_size
|
|
2080
2471
|
return total_size
|
|
2081
2472
|
return self.lstat(objpath).st_size
|
|
2082
2473
|
|
|
@@ -2084,8 +2475,8 @@ class OSExtended(System):
|
|
|
2084
2475
|
"""Normalises path name of **dirpath** and recursively creates this directory."""
|
|
2085
2476
|
normdir = self.path.normpath(self.path.expanduser(dirpath))
|
|
2086
2477
|
if normdir and not self.path.isdir(normdir):
|
|
2087
|
-
logger.debug(
|
|
2088
|
-
self.stderr(
|
|
2478
|
+
logger.debug("Cocooning directory %s", normdir)
|
|
2479
|
+
self.stderr("mkdir", normdir)
|
|
2089
2480
|
try:
|
|
2090
2481
|
self.makedirs(normdir)
|
|
2091
2482
|
return True
|
|
@@ -2102,17 +2493,17 @@ class OSExtended(System):
|
|
|
2102
2493
|
"""Normalises path name of ``destination`` and creates **destination**'s directory."""
|
|
2103
2494
|
return self.mkdir(self.path.dirname(self.path.expanduser(destination)))
|
|
2104
2495
|
|
|
2105
|
-
_SAFE_SUFFIX_RE = re.compile(
|
|
2496
|
+
_SAFE_SUFFIX_RE = re.compile("_[a-f0-9]{32}$")
|
|
2106
2497
|
|
|
2107
2498
|
def safe_filesuffix(self):
|
|
2108
2499
|
"""Returns a file suffix that should be unique across the system."""
|
|
2109
|
-
return
|
|
2500
|
+
return "_" + uuid.uuid1().hex
|
|
2110
2501
|
|
|
2111
2502
|
def safe_fileaddsuffix(self, name):
|
|
2112
2503
|
"""Returns a file path that will look like name + a unique suffix."""
|
|
2113
2504
|
d_name = self.path.dirname(name)
|
|
2114
2505
|
b_name = self.path.basename(name)
|
|
2115
|
-
b_name = self._SAFE_SUFFIX_RE.sub(
|
|
2506
|
+
b_name = self._SAFE_SUFFIX_RE.sub("", b_name)
|
|
2116
2507
|
return self.path.join(d_name, b_name + self.safe_filesuffix())
|
|
2117
2508
|
|
|
2118
2509
|
def _validate_symlink_below(self, symlink, valid_below):
|
|
@@ -2125,20 +2516,26 @@ class OSExtended(System):
|
|
|
2125
2516
|
"""
|
|
2126
2517
|
link_to = self._os.readlink(symlink)
|
|
2127
2518
|
# Is it relative ?
|
|
2128
|
-
if re.match(
|
|
2129
|
-
|
|
2519
|
+
if re.match(
|
|
2520
|
+
"^([^{0:s}]|..{0:s}|.{0:s})".format(re.escape(os.path.sep)),
|
|
2521
|
+
link_to,
|
|
2522
|
+
):
|
|
2130
2523
|
symlink_dir = self.path.realpath(
|
|
2131
|
-
self.path.abspath(
|
|
2132
|
-
self.path.dirname(symlink)
|
|
2133
|
-
)
|
|
2524
|
+
self.path.abspath(self.path.dirname(symlink))
|
|
2134
2525
|
)
|
|
2135
2526
|
abspath_to = self.path.normpath(
|
|
2136
2527
|
self.path.join(symlink_dir, link_to)
|
|
2137
2528
|
)
|
|
2138
2529
|
# Valid ?
|
|
2139
|
-
valid =
|
|
2140
|
-
|
|
2141
|
-
|
|
2530
|
+
valid = (
|
|
2531
|
+
self.path.commonprefix([valid_below, abspath_to])
|
|
2532
|
+
== valid_below
|
|
2533
|
+
)
|
|
2534
|
+
return (
|
|
2535
|
+
self.path.relpath(abspath_to, start=symlink_dir)
|
|
2536
|
+
if valid
|
|
2537
|
+
else None
|
|
2538
|
+
)
|
|
2142
2539
|
else:
|
|
2143
2540
|
return None
|
|
2144
2541
|
|
|
@@ -2150,10 +2547,11 @@ class OSExtended(System):
|
|
|
2150
2547
|
|
|
2151
2548
|
The destination directory must not already exist.
|
|
2152
2549
|
"""
|
|
2153
|
-
self.stderr(
|
|
2550
|
+
self.stderr("_copydatatree", src, dst)
|
|
2154
2551
|
with self.mute_stderr():
|
|
2155
|
-
keep_symlinks_below =
|
|
2156
|
-
|
|
2552
|
+
keep_symlinks_below = keep_symlinks_below or self.path.realpath(
|
|
2553
|
+
self.path.abspath(src)
|
|
2554
|
+
)
|
|
2157
2555
|
names = self._os.listdir(src)
|
|
2158
2556
|
self._os.makedirs(dst)
|
|
2159
2557
|
errors = []
|
|
@@ -2162,14 +2560,19 @@ class OSExtended(System):
|
|
|
2162
2560
|
dstname = self._os.path.join(dst, name)
|
|
2163
2561
|
try:
|
|
2164
2562
|
if self.path.isdir(srcname):
|
|
2165
|
-
self._copydatatree(
|
|
2166
|
-
|
|
2563
|
+
self._copydatatree(
|
|
2564
|
+
srcname,
|
|
2565
|
+
dstname,
|
|
2566
|
+
keep_symlinks_below=keep_symlinks_below,
|
|
2567
|
+
)
|
|
2167
2568
|
elif self._os.path.islink(srcname):
|
|
2168
|
-
linkto = self._validate_symlink_below(
|
|
2569
|
+
linkto = self._validate_symlink_below(
|
|
2570
|
+
srcname, keep_symlinks_below
|
|
2571
|
+
)
|
|
2169
2572
|
if linkto is not None:
|
|
2170
2573
|
self._os.symlink(linkto, dstname)
|
|
2171
2574
|
else:
|
|
2172
|
-
|
|
2575
|
+
self._sh.copyfile(srcname, dstname)
|
|
2173
2576
|
else:
|
|
2174
2577
|
# Will raise a SpecialFileError for unsupported file types
|
|
2175
2578
|
self._sh.copyfile(srcname, dstname)
|
|
@@ -2191,7 +2594,7 @@ class OSExtended(System):
|
|
|
2191
2594
|
"""
|
|
2192
2595
|
source = self.path.expanduser(source)
|
|
2193
2596
|
destination = self.path.expanduser(destination)
|
|
2194
|
-
self.stderr(
|
|
2597
|
+
self.stderr("rawcp", source, destination)
|
|
2195
2598
|
tmp = self.safe_fileaddsuffix(destination)
|
|
2196
2599
|
if self.path.isdir(source):
|
|
2197
2600
|
self._copydatatree(source, tmp)
|
|
@@ -2218,29 +2621,35 @@ class OSExtended(System):
|
|
|
2218
2621
|
If **destination** is a real-word file name (i.e. not e File-like object),
|
|
2219
2622
|
the operation is atomic.
|
|
2220
2623
|
"""
|
|
2221
|
-
self.stderr(
|
|
2624
|
+
self.stderr("hybridcp", source, destination)
|
|
2222
2625
|
if isinstance(source, str):
|
|
2223
2626
|
if not self.path.exists(source):
|
|
2224
2627
|
if not silent:
|
|
2225
|
-
logger.error(
|
|
2628
|
+
logger.error("Missing source %s", source)
|
|
2226
2629
|
return False
|
|
2227
|
-
source = open(self.path.expanduser(source),
|
|
2630
|
+
source = open(self.path.expanduser(source), "rb")
|
|
2228
2631
|
xsource = True
|
|
2229
2632
|
else:
|
|
2230
2633
|
xsource = False
|
|
2231
2634
|
try:
|
|
2232
2635
|
source.seek(0)
|
|
2233
2636
|
except AttributeError:
|
|
2234
|
-
logger.warning(
|
|
2637
|
+
logger.warning(
|
|
2638
|
+
"Could not rewind io source before cp: " + str(source)
|
|
2639
|
+
)
|
|
2235
2640
|
if isinstance(destination, str):
|
|
2236
2641
|
if self.filecocoon(destination):
|
|
2237
2642
|
# Write to a temp file
|
|
2238
2643
|
original_dest = self.path.expanduser(destination)
|
|
2239
|
-
tmp_dest = self.safe_fileaddsuffix(
|
|
2240
|
-
|
|
2644
|
+
tmp_dest = self.safe_fileaddsuffix(
|
|
2645
|
+
self.path.expanduser(destination)
|
|
2646
|
+
)
|
|
2647
|
+
destination = open(tmp_dest, "wb")
|
|
2241
2648
|
xdestination = True
|
|
2242
2649
|
else:
|
|
2243
|
-
logger.error(
|
|
2650
|
+
logger.error(
|
|
2651
|
+
"Could not create a cocoon for file %s", destination
|
|
2652
|
+
)
|
|
2244
2653
|
return False
|
|
2245
2654
|
else:
|
|
2246
2655
|
destination.seek(0)
|
|
@@ -2253,8 +2662,13 @@ class OSExtended(System):
|
|
|
2253
2662
|
if xdestination:
|
|
2254
2663
|
destination.close()
|
|
2255
2664
|
# Move the tmp_file to the real destination
|
|
2256
|
-
if not self.move(
|
|
2257
|
-
|
|
2665
|
+
if not self.move(
|
|
2666
|
+
tmp_dest, original_dest
|
|
2667
|
+
): # Move is atomic for a file
|
|
2668
|
+
logger.error(
|
|
2669
|
+
"Cannot move the tmp file to the final destination %s",
|
|
2670
|
+
original_dest,
|
|
2671
|
+
)
|
|
2258
2672
|
return False
|
|
2259
2673
|
return rc
|
|
2260
2674
|
|
|
@@ -2265,7 +2679,7 @@ class OSExtended(System):
|
|
|
2265
2679
|
return st1.st_dev == st2.st_dev and not self.path.islink(path1)
|
|
2266
2680
|
|
|
2267
2681
|
def _rawcp_instead_of_hardlink(self, source, destination, securecopy=True):
|
|
2268
|
-
self.stderr(
|
|
2682
|
+
self.stderr("rawcp_instead_of_hardlink", source, destination)
|
|
2269
2683
|
if securecopy:
|
|
2270
2684
|
rc = self.rawcp(source, destination)
|
|
2271
2685
|
else:
|
|
@@ -2294,19 +2708,29 @@ class OSExtended(System):
|
|
|
2294
2708
|
except OSError as e:
|
|
2295
2709
|
if e.errno == errno.EMLINK:
|
|
2296
2710
|
# Too many links
|
|
2297
|
-
logger.warning(
|
|
2711
|
+
logger.warning(
|
|
2712
|
+
"Too many links for the source file (%s).", source
|
|
2713
|
+
)
|
|
2298
2714
|
if self.usr_file(source):
|
|
2299
|
-
rc = self._rawcp_instead_of_hardlink(
|
|
2715
|
+
rc = self._rawcp_instead_of_hardlink(
|
|
2716
|
+
source, destination, securecopy=securecopy
|
|
2717
|
+
)
|
|
2300
2718
|
if rc:
|
|
2301
2719
|
try:
|
|
2302
|
-
logger.warning(
|
|
2720
|
+
logger.warning(
|
|
2721
|
+
"Replacing the orignal file with a copy..."
|
|
2722
|
+
)
|
|
2303
2723
|
self.move(destination, source)
|
|
2304
2724
|
except OSError as ebis:
|
|
2305
2725
|
if ebis.errno == errno.EACCES:
|
|
2306
2726
|
# Permission denied
|
|
2307
|
-
logger.warning(
|
|
2308
|
-
|
|
2309
|
-
|
|
2727
|
+
logger.warning(
|
|
2728
|
+
"No permissions to create a copy of the source file (%s)",
|
|
2729
|
+
source,
|
|
2730
|
+
)
|
|
2731
|
+
logger.warning(
|
|
2732
|
+
"Going on with the copy instead of the link..."
|
|
2733
|
+
)
|
|
2310
2734
|
else:
|
|
2311
2735
|
raise
|
|
2312
2736
|
else:
|
|
@@ -2319,9 +2743,15 @@ class OSExtended(System):
|
|
|
2319
2743
|
rc = self.path.samefile(source, destination)
|
|
2320
2744
|
return rc
|
|
2321
2745
|
|
|
2322
|
-
def hardlink(
|
|
2323
|
-
|
|
2324
|
-
|
|
2746
|
+
def hardlink(
|
|
2747
|
+
self,
|
|
2748
|
+
source,
|
|
2749
|
+
destination,
|
|
2750
|
+
link_threshold=0,
|
|
2751
|
+
readonly=True,
|
|
2752
|
+
securecopy=True,
|
|
2753
|
+
keep_symlinks_below=None,
|
|
2754
|
+
):
|
|
2325
2755
|
"""Create hardlinks for both single files or directories.
|
|
2326
2756
|
|
|
2327
2757
|
:param int link_threshold: if the source file size is smaller than
|
|
@@ -2337,10 +2767,17 @@ class OSExtended(System):
|
|
|
2337
2767
|
directory (if omitted, **source** is used)
|
|
2338
2768
|
"""
|
|
2339
2769
|
if self.path.isdir(source):
|
|
2340
|
-
self.stderr(
|
|
2341
|
-
|
|
2342
|
-
|
|
2343
|
-
|
|
2770
|
+
self.stderr(
|
|
2771
|
+
"hardlink",
|
|
2772
|
+
source,
|
|
2773
|
+
destination,
|
|
2774
|
+
"#",
|
|
2775
|
+
"directory,",
|
|
2776
|
+
"readonly={!s}".format(readonly),
|
|
2777
|
+
)
|
|
2778
|
+
keep_symlinks_below = keep_symlinks_below or self.path.realpath(
|
|
2779
|
+
self.path.abspath(source)
|
|
2780
|
+
)
|
|
2344
2781
|
with self.mute_stderr():
|
|
2345
2782
|
# Mimics 'cp -al'
|
|
2346
2783
|
names = self._os.listdir(source)
|
|
@@ -2350,30 +2787,53 @@ class OSExtended(System):
|
|
|
2350
2787
|
srcname = self._os.path.join(source, name)
|
|
2351
2788
|
dstname = self._os.path.join(destination, name)
|
|
2352
2789
|
if self._os.path.islink(srcname):
|
|
2353
|
-
linkto = self._validate_symlink_below(
|
|
2790
|
+
linkto = self._validate_symlink_below(
|
|
2791
|
+
srcname, keep_symlinks_below
|
|
2792
|
+
)
|
|
2354
2793
|
if linkto is None:
|
|
2355
|
-
link_target = self.path.join(
|
|
2356
|
-
|
|
2357
|
-
|
|
2358
|
-
|
|
2359
|
-
|
|
2360
|
-
|
|
2794
|
+
link_target = self.path.join(
|
|
2795
|
+
self.path.dirname(srcname),
|
|
2796
|
+
self._os.readlink(srcname),
|
|
2797
|
+
)
|
|
2798
|
+
rc = self.hardlink(
|
|
2799
|
+
link_target,
|
|
2800
|
+
dstname,
|
|
2801
|
+
link_threshold=link_threshold,
|
|
2802
|
+
readonly=readonly,
|
|
2803
|
+
securecopy=securecopy,
|
|
2804
|
+
keep_symlinks_below=keep_symlinks_below,
|
|
2805
|
+
)
|
|
2361
2806
|
else:
|
|
2362
2807
|
self._os.symlink(linkto, dstname)
|
|
2363
2808
|
elif self.path.isdir(srcname):
|
|
2364
|
-
rc = self.hardlink(
|
|
2365
|
-
|
|
2366
|
-
|
|
2367
|
-
|
|
2809
|
+
rc = self.hardlink(
|
|
2810
|
+
srcname,
|
|
2811
|
+
dstname,
|
|
2812
|
+
link_threshold=link_threshold,
|
|
2813
|
+
readonly=readonly,
|
|
2814
|
+
securecopy=securecopy,
|
|
2815
|
+
keep_symlinks_below=keep_symlinks_below,
|
|
2816
|
+
)
|
|
2368
2817
|
else:
|
|
2369
|
-
if
|
|
2370
|
-
|
|
2818
|
+
if (
|
|
2819
|
+
link_threshold
|
|
2820
|
+
and self.size(srcname) < link_threshold
|
|
2821
|
+
):
|
|
2822
|
+
rc = self._rawcp_instead_of_hardlink(
|
|
2823
|
+
srcname, dstname, securecopy=securecopy
|
|
2824
|
+
)
|
|
2371
2825
|
else:
|
|
2372
|
-
rc = self._safe_hardlink(
|
|
2826
|
+
rc = self._safe_hardlink(
|
|
2827
|
+
srcname, dstname, securecopy=securecopy
|
|
2828
|
+
)
|
|
2373
2829
|
if readonly and rc:
|
|
2374
2830
|
self.readonly(dstname)
|
|
2375
2831
|
if not rc:
|
|
2376
|
-
logger.error(
|
|
2832
|
+
logger.error(
|
|
2833
|
+
"Error while processing %s (rc=%s)",
|
|
2834
|
+
srcname,
|
|
2835
|
+
str(rc),
|
|
2836
|
+
)
|
|
2377
2837
|
break
|
|
2378
2838
|
if rc:
|
|
2379
2839
|
self._sh.copystat(source, destination)
|
|
@@ -2381,16 +2841,27 @@ class OSExtended(System):
|
|
|
2381
2841
|
return rc
|
|
2382
2842
|
else:
|
|
2383
2843
|
if link_threshold and self.size(source) < link_threshold:
|
|
2384
|
-
rc = self._rawcp_instead_of_hardlink(
|
|
2844
|
+
rc = self._rawcp_instead_of_hardlink(
|
|
2845
|
+
source, destination, securecopy=securecopy
|
|
2846
|
+
)
|
|
2385
2847
|
else:
|
|
2386
|
-
self.stderr(
|
|
2387
|
-
rc = self._safe_hardlink(
|
|
2848
|
+
self.stderr("hardlink", source, destination)
|
|
2849
|
+
rc = self._safe_hardlink(
|
|
2850
|
+
source, destination, securecopy=securecopy
|
|
2851
|
+
)
|
|
2388
2852
|
if readonly and rc:
|
|
2389
2853
|
self.readonly(destination)
|
|
2390
2854
|
return rc
|
|
2391
2855
|
|
|
2392
|
-
def _smartcp_cross_users_links_fallback(
|
|
2393
|
-
|
|
2856
|
+
def _smartcp_cross_users_links_fallback(
|
|
2857
|
+
self,
|
|
2858
|
+
source,
|
|
2859
|
+
destination,
|
|
2860
|
+
smartcp_threshold,
|
|
2861
|
+
silent,
|
|
2862
|
+
exc,
|
|
2863
|
+
tmp_destination=None,
|
|
2864
|
+
):
|
|
2394
2865
|
"""Catch errors related to Kernel configuration."""
|
|
2395
2866
|
if (exc.errno == errno.EPERM) and not self.usr_file(source):
|
|
2396
2867
|
# This is expected to fail if the fs.protected_hardlinks
|
|
@@ -2400,8 +2871,12 @@ class OSExtended(System):
|
|
|
2400
2871
|
logger.info("Force System's allow_cross_users_links to False")
|
|
2401
2872
|
self.allow_cross_users_links = False
|
|
2402
2873
|
logger.info("Re-running the smartcp command")
|
|
2403
|
-
return self.smartcp(
|
|
2404
|
-
|
|
2874
|
+
return self.smartcp(
|
|
2875
|
+
source,
|
|
2876
|
+
destination,
|
|
2877
|
+
smartcp_threshold=smartcp_threshold,
|
|
2878
|
+
silent=silent,
|
|
2879
|
+
)
|
|
2405
2880
|
else:
|
|
2406
2881
|
raise
|
|
2407
2882
|
|
|
@@ -2415,30 +2890,39 @@ class OSExtended(System):
|
|
|
2415
2890
|
When working on a file, the operation is atomic. When working on a
|
|
2416
2891
|
directory some restrictions apply (see :meth:`rawcp`)
|
|
2417
2892
|
"""
|
|
2418
|
-
self.stderr(
|
|
2893
|
+
self.stderr("smartcp", source, destination)
|
|
2419
2894
|
if not isinstance(source, str) or not isinstance(destination, str):
|
|
2420
2895
|
return self.hybridcp(source, destination)
|
|
2421
2896
|
source = self.path.expanduser(source)
|
|
2422
2897
|
if not self.path.exists(source):
|
|
2423
2898
|
if not silent:
|
|
2424
|
-
logger.error(
|
|
2899
|
+
logger.error("Missing source %s", source)
|
|
2425
2900
|
return False
|
|
2426
2901
|
if self.filecocoon(destination):
|
|
2427
2902
|
destination = self.path.expanduser(destination)
|
|
2428
2903
|
if self.path.islink(source):
|
|
2429
2904
|
# Solve the symbolic link: this may avoid a rawcp
|
|
2430
2905
|
source = self.path.realpath(source)
|
|
2431
|
-
if
|
|
2432
|
-
|
|
2906
|
+
if self.is_samefs(source, destination) and (
|
|
2907
|
+
self.allow_cross_users_links or self.usr_file(source)
|
|
2908
|
+
):
|
|
2433
2909
|
tmp_destination = self.safe_fileaddsuffix(destination)
|
|
2434
2910
|
if self.path.isdir(source):
|
|
2435
2911
|
try:
|
|
2436
|
-
rc = self.hardlink(
|
|
2437
|
-
|
|
2912
|
+
rc = self.hardlink(
|
|
2913
|
+
source,
|
|
2914
|
+
tmp_destination,
|
|
2915
|
+
link_threshold=smartcp_threshold,
|
|
2916
|
+
securecopy=False,
|
|
2917
|
+
)
|
|
2438
2918
|
except OSError as e:
|
|
2439
2919
|
rc = self._smartcp_cross_users_links_fallback(
|
|
2440
|
-
source,
|
|
2441
|
-
|
|
2920
|
+
source,
|
|
2921
|
+
destination,
|
|
2922
|
+
smartcp_threshold,
|
|
2923
|
+
silent,
|
|
2924
|
+
e,
|
|
2925
|
+
tmp_destination=tmp_destination,
|
|
2442
2926
|
)
|
|
2443
2927
|
else:
|
|
2444
2928
|
if rc:
|
|
@@ -2446,44 +2930,71 @@ class OSExtended(System):
|
|
|
2446
2930
|
with self.secure_directory_move(destination):
|
|
2447
2931
|
rc = self.move(tmp_destination, destination)
|
|
2448
2932
|
if not rc:
|
|
2449
|
-
logger.error(
|
|
2450
|
-
|
|
2451
|
-
|
|
2933
|
+
logger.error(
|
|
2934
|
+
"Cannot move the tmp directory to the final destination %s",
|
|
2935
|
+
destination,
|
|
2936
|
+
)
|
|
2937
|
+
self.remove(
|
|
2938
|
+
tmp_destination
|
|
2939
|
+
) # Anyway, try to clean-up things
|
|
2452
2940
|
else:
|
|
2453
|
-
logger.error(
|
|
2454
|
-
|
|
2941
|
+
logger.error(
|
|
2942
|
+
"Cannot copy the data to the tmp directory %s",
|
|
2943
|
+
tmp_destination,
|
|
2944
|
+
)
|
|
2945
|
+
self.remove(
|
|
2946
|
+
tmp_destination
|
|
2947
|
+
) # Anyway, try to clean-up things
|
|
2455
2948
|
return rc
|
|
2456
2949
|
else:
|
|
2457
2950
|
try:
|
|
2458
|
-
rc = self.hardlink(
|
|
2459
|
-
|
|
2951
|
+
rc = self.hardlink(
|
|
2952
|
+
source,
|
|
2953
|
+
tmp_destination,
|
|
2954
|
+
link_threshold=smartcp_threshold,
|
|
2955
|
+
securecopy=False,
|
|
2956
|
+
)
|
|
2460
2957
|
except OSError as e:
|
|
2461
|
-
rc = self._smartcp_cross_users_links_fallback(
|
|
2462
|
-
|
|
2958
|
+
rc = self._smartcp_cross_users_links_fallback(
|
|
2959
|
+
source, destination, smartcp_threshold, silent, e
|
|
2960
|
+
)
|
|
2463
2961
|
else:
|
|
2464
|
-
rc = rc and self.move(
|
|
2962
|
+
rc = rc and self.move(
|
|
2963
|
+
tmp_destination, destination
|
|
2964
|
+
) # Move is atomic for a file
|
|
2465
2965
|
# On some systems, the temporary file may remain (if the
|
|
2466
2966
|
# destination's inode is identical to the tmp_destination's
|
|
2467
2967
|
# inode). The following call to remove will remove leftovers.
|
|
2468
2968
|
self.remove(tmp_destination)
|
|
2469
2969
|
return rc
|
|
2470
2970
|
else:
|
|
2471
|
-
rc = self.rawcp(
|
|
2971
|
+
rc = self.rawcp(
|
|
2972
|
+
source, destination
|
|
2973
|
+
) # Rawcp is atomic as much as possible
|
|
2472
2974
|
if rc:
|
|
2473
2975
|
if self.path.isdir(destination):
|
|
2474
2976
|
for copiedfile in self.ffind(destination):
|
|
2475
|
-
if not self.path.islink(
|
|
2977
|
+
if not self.path.islink(
|
|
2978
|
+
copiedfile
|
|
2979
|
+
): # This make no sense to chmod symlinks
|
|
2476
2980
|
self.chmod(copiedfile, 0o444)
|
|
2477
2981
|
else:
|
|
2478
2982
|
self.readonly(destination)
|
|
2479
2983
|
return rc
|
|
2480
2984
|
else:
|
|
2481
|
-
logger.error(
|
|
2985
|
+
logger.error("Could not create a cocoon for file %s", destination)
|
|
2482
2986
|
return False
|
|
2483
2987
|
|
|
2484
2988
|
@fmtshcmd
|
|
2485
|
-
def cp(
|
|
2486
|
-
|
|
2989
|
+
def cp(
|
|
2990
|
+
self,
|
|
2991
|
+
source,
|
|
2992
|
+
destination,
|
|
2993
|
+
intent="inout",
|
|
2994
|
+
smartcp=True,
|
|
2995
|
+
smartcp_threshold=0,
|
|
2996
|
+
silent=False,
|
|
2997
|
+
):
|
|
2487
2998
|
"""Copy the **source** file to a safe **destination**.
|
|
2488
2999
|
|
|
2489
3000
|
:param source: The source of data (either a path to file or a
|
|
@@ -2504,27 +3015,31 @@ class OSExtended(System):
|
|
|
2504
3015
|
|
|
2505
3016
|
The fastest option should be used...
|
|
2506
3017
|
"""
|
|
2507
|
-
self.stderr(
|
|
3018
|
+
self.stderr("cp", source, destination)
|
|
2508
3019
|
if not isinstance(source, str) or not isinstance(destination, str):
|
|
2509
3020
|
return self.hybridcp(source, destination, silent=silent)
|
|
2510
3021
|
if not self.path.exists(source):
|
|
2511
3022
|
if not silent:
|
|
2512
|
-
logger.error(
|
|
3023
|
+
logger.error("Missing source %s", source)
|
|
2513
3024
|
return False
|
|
2514
|
-
if smartcp and intent ==
|
|
2515
|
-
return self.smartcp(
|
|
2516
|
-
|
|
3025
|
+
if smartcp and intent == "in":
|
|
3026
|
+
return self.smartcp(
|
|
3027
|
+
source,
|
|
3028
|
+
destination,
|
|
3029
|
+
smartcp_threshold=smartcp_threshold,
|
|
3030
|
+
silent=silent,
|
|
3031
|
+
)
|
|
2517
3032
|
if self.filecocoon(destination):
|
|
2518
3033
|
return self.rawcp(source, destination)
|
|
2519
3034
|
else:
|
|
2520
|
-
logger.error(
|
|
3035
|
+
logger.error("Could not create a cocoon for file %s", destination)
|
|
2521
3036
|
return False
|
|
2522
3037
|
|
|
2523
3038
|
def glob(self, *args):
|
|
2524
3039
|
"""Glob file system entries according to ``args``. Returns a list."""
|
|
2525
3040
|
entries = []
|
|
2526
3041
|
for entry in args:
|
|
2527
|
-
if entry.startswith(
|
|
3042
|
+
if entry.startswith(":"):
|
|
2528
3043
|
entries.append(entry[1:])
|
|
2529
3044
|
else:
|
|
2530
3045
|
entries.extend(glob.glob(self.path.expanduser(entry)))
|
|
@@ -2544,15 +3059,23 @@ class OSExtended(System):
|
|
|
2544
3059
|
"""
|
|
2545
3060
|
safe = True
|
|
2546
3061
|
if len(thispath.split(self._os.sep)) < self._rmtreemin + 1:
|
|
2547
|
-
logger.warning(
|
|
3062
|
+
logger.warning(
|
|
3063
|
+
"Unsafe starting point depth %s (min is %s)",
|
|
3064
|
+
thispath,
|
|
3065
|
+
self._rmtreemin,
|
|
3066
|
+
)
|
|
2548
3067
|
safe = False
|
|
2549
3068
|
else:
|
|
2550
3069
|
for safepack in safedirs:
|
|
2551
3070
|
(safedir, d) = safepack
|
|
2552
3071
|
rp = self.path.relpath(thispath, safedir)
|
|
2553
|
-
if not rp.startswith(
|
|
3072
|
+
if not rp.startswith(".."):
|
|
2554
3073
|
if len(rp.split(self._os.sep)) < d:
|
|
2555
|
-
logger.warning(
|
|
3074
|
+
logger.warning(
|
|
3075
|
+
"Unsafe access to %s relative to %s",
|
|
3076
|
+
thispath,
|
|
3077
|
+
safedir,
|
|
3078
|
+
)
|
|
2556
3079
|
safe = False
|
|
2557
3080
|
return safe
|
|
2558
3081
|
|
|
@@ -2565,43 +3088,47 @@ class OSExtended(System):
|
|
|
2565
3088
|
if isinstance(pathlist, str):
|
|
2566
3089
|
pathlist = [pathlist]
|
|
2567
3090
|
for pname in pathlist:
|
|
2568
|
-
for entry in filter(
|
|
3091
|
+
for entry in filter(
|
|
3092
|
+
lambda x: self.safepath(x, safedirs), self.glob(pname)
|
|
3093
|
+
):
|
|
2569
3094
|
ok = self.remove(entry) and ok
|
|
2570
3095
|
return ok
|
|
2571
3096
|
|
|
2572
3097
|
def _globcmd(self, cmd, args, **kw):
|
|
2573
3098
|
"""Globbing files or directories as arguments before running ``cmd``."""
|
|
2574
|
-
cmd.extend([opt for opt in args if opt.startswith(
|
|
3099
|
+
cmd.extend([opt for opt in args if opt.startswith("-")])
|
|
2575
3100
|
cmdlen = len(cmd)
|
|
2576
3101
|
cmdargs = False
|
|
2577
|
-
globtries = [
|
|
3102
|
+
globtries = [
|
|
3103
|
+
self.path.expanduser(x) for x in args if not x.startswith("-")
|
|
3104
|
+
]
|
|
2578
3105
|
for pname in globtries:
|
|
2579
3106
|
cmdargs = True
|
|
2580
3107
|
cmd.extend(self.glob(pname))
|
|
2581
3108
|
if cmdargs and len(cmd) == cmdlen:
|
|
2582
|
-
logger.warning(
|
|
3109
|
+
logger.warning("Could not find any matching pattern %s", globtries)
|
|
2583
3110
|
return False
|
|
2584
3111
|
else:
|
|
2585
|
-
kw.setdefault(
|
|
3112
|
+
kw.setdefault("ok", [0])
|
|
2586
3113
|
return self.spawn(cmd, **kw)
|
|
2587
3114
|
|
|
2588
3115
|
@_kw2spawn
|
|
2589
3116
|
def wc(self, *args, **kw):
|
|
2590
3117
|
"""Word count on globbed files."""
|
|
2591
|
-
return self._globcmd([
|
|
3118
|
+
return self._globcmd(["wc"], args, **kw)
|
|
2592
3119
|
|
|
2593
3120
|
@_kw2spawn
|
|
2594
3121
|
def ls(self, *args, **kw):
|
|
2595
3122
|
"""Clone of the eponymous unix command."""
|
|
2596
|
-
return self._globcmd([
|
|
3123
|
+
return self._globcmd(["ls"], args, **kw)
|
|
2597
3124
|
|
|
2598
3125
|
@_kw2spawn
|
|
2599
3126
|
def ll(self, *args, **kw):
|
|
2600
3127
|
"""Clone of the eponymous unix alias (ls -l)."""
|
|
2601
|
-
kw[
|
|
2602
|
-
llresult = self._globcmd([
|
|
3128
|
+
kw["output"] = True
|
|
3129
|
+
llresult = self._globcmd(["ls", "-l"], args, **kw)
|
|
2603
3130
|
if llresult:
|
|
2604
|
-
for lline in [x for x in llresult if not x.startswith(
|
|
3131
|
+
for lline in [x for x in llresult if not x.startswith("total")]:
|
|
2605
3132
|
print(lline)
|
|
2606
3133
|
else:
|
|
2607
3134
|
return False
|
|
@@ -2609,25 +3136,25 @@ class OSExtended(System):
|
|
|
2609
3136
|
@_kw2spawn
|
|
2610
3137
|
def dir(self, *args, **kw):
|
|
2611
3138
|
"""Proxy to ``ls('-l')``."""
|
|
2612
|
-
return self._globcmd([
|
|
3139
|
+
return self._globcmd(["ls", "-l"], args, **kw)
|
|
2613
3140
|
|
|
2614
3141
|
@_kw2spawn
|
|
2615
3142
|
def cat(self, *args, **kw):
|
|
2616
3143
|
"""Clone of the eponymous unix command."""
|
|
2617
|
-
return self._globcmd([
|
|
3144
|
+
return self._globcmd(["cat"], args, **kw)
|
|
2618
3145
|
|
|
2619
3146
|
@fmtshcmd
|
|
2620
3147
|
@_kw2spawn
|
|
2621
3148
|
def diff(self, *args, **kw):
|
|
2622
3149
|
"""Clone of the eponymous unix command."""
|
|
2623
|
-
kw.setdefault(
|
|
2624
|
-
kw.setdefault(
|
|
2625
|
-
return self._globcmd([
|
|
3150
|
+
kw.setdefault("ok", [0, 1])
|
|
3151
|
+
kw.setdefault("output", False)
|
|
3152
|
+
return self._globcmd(["cmp"], args, **kw)
|
|
2626
3153
|
|
|
2627
3154
|
@_kw2spawn
|
|
2628
3155
|
def rmglob(self, *args, **kw):
|
|
2629
3156
|
"""Wrapper of the shell's ``rm`` command through the :meth:`globcmd` method."""
|
|
2630
|
-
return self._globcmd([
|
|
3157
|
+
return self._globcmd(["rm"], args, **kw)
|
|
2631
3158
|
|
|
2632
3159
|
@fmtshcmd
|
|
2633
3160
|
def move(self, source, destination):
|
|
@@ -2636,20 +3163,23 @@ class OSExtended(System):
|
|
|
2636
3163
|
:param str source: The source object (file, directory, ...)
|
|
2637
3164
|
:param str destination: The destination object (file, directory, ...)
|
|
2638
3165
|
"""
|
|
2639
|
-
self.stderr(
|
|
3166
|
+
self.stderr("move", source, destination)
|
|
2640
3167
|
try:
|
|
2641
3168
|
self._sh.move(source, destination)
|
|
2642
3169
|
except Exception:
|
|
2643
|
-
logger.critical(
|
|
3170
|
+
logger.critical("Could not move <%s> to <%s>", source, destination)
|
|
2644
3171
|
raise
|
|
2645
3172
|
else:
|
|
2646
3173
|
return True
|
|
2647
3174
|
|
|
2648
3175
|
@contextlib.contextmanager
|
|
2649
3176
|
def secure_directory_move(self, destination):
|
|
2650
|
-
with self.lockdir_context(
|
|
2651
|
-
|
|
2652
|
-
|
|
3177
|
+
with self.lockdir_context(
|
|
3178
|
+
destination + ".vortex-lockdir", sloppy=True
|
|
3179
|
+
):
|
|
3180
|
+
do_cleanup = isinstance(destination, str) and self.path.exists(
|
|
3181
|
+
destination
|
|
3182
|
+
)
|
|
2653
3183
|
if do_cleanup:
|
|
2654
3184
|
# Warning: Not an atomic portion of code (sorry)
|
|
2655
3185
|
tmp_destination = self.safe_fileaddsuffix(destination)
|
|
@@ -2668,7 +3198,7 @@ class OSExtended(System):
|
|
|
2668
3198
|
:param source: The source object (file, directory, File-like object, ...)
|
|
2669
3199
|
:param destination: The destination object (file, directory, File-like object, ...)
|
|
2670
3200
|
"""
|
|
2671
|
-
self.stderr(
|
|
3201
|
+
self.stderr("mv", source, destination)
|
|
2672
3202
|
if not isinstance(source, str) or not isinstance(destination, str):
|
|
2673
3203
|
self.hybridcp(source, destination)
|
|
2674
3204
|
if isinstance(source, str):
|
|
@@ -2679,13 +3209,13 @@ class OSExtended(System):
|
|
|
2679
3209
|
@_kw2spawn
|
|
2680
3210
|
def mvglob(self, *args):
|
|
2681
3211
|
"""Wrapper of the shell's ``mv`` command through the :meth:`globcmd` method."""
|
|
2682
|
-
return self._globcmd([
|
|
3212
|
+
return self._globcmd(["mv"], args)
|
|
2683
3213
|
|
|
2684
3214
|
def listdir(self, *args):
|
|
2685
3215
|
"""Proxy to standard :mod:`os` directory listing function."""
|
|
2686
3216
|
if not args:
|
|
2687
|
-
args = (
|
|
2688
|
-
self.stderr(
|
|
3217
|
+
args = (".",)
|
|
3218
|
+
self.stderr("listdir", *args)
|
|
2689
3219
|
return self._os.listdir(self.path.expanduser(args[0]))
|
|
2690
3220
|
|
|
2691
3221
|
def pyls(self, *args):
|
|
@@ -2694,10 +3224,10 @@ class OSExtended(System):
|
|
|
2694
3224
|
:meth:`ls` method except that that shell's ``ls`` command is not actually
|
|
2695
3225
|
called.
|
|
2696
3226
|
"""
|
|
2697
|
-
rl = [x for x in args if not x.startswith(
|
|
3227
|
+
rl = [x for x in args if not x.startswith("-")]
|
|
2698
3228
|
if not rl:
|
|
2699
|
-
rl.append(
|
|
2700
|
-
self.stderr(
|
|
3229
|
+
rl.append("*")
|
|
3230
|
+
self.stderr("pyls", *rl)
|
|
2701
3231
|
return self.glob(*rl)
|
|
2702
3232
|
|
|
2703
3233
|
def ldirs(self, *args):
|
|
@@ -2706,23 +3236,23 @@ class OSExtended(System):
|
|
|
2706
3236
|
:meth:`ls` method except that that shell's ``ls`` command is not actually
|
|
2707
3237
|
called.
|
|
2708
3238
|
"""
|
|
2709
|
-
rl = [x for x in args if not x.startswith(
|
|
3239
|
+
rl = [x for x in args if not x.startswith("-")]
|
|
2710
3240
|
if not rl:
|
|
2711
|
-
rl.append(
|
|
2712
|
-
self.stderr(
|
|
3241
|
+
rl.append("*")
|
|
3242
|
+
self.stderr("ldirs", *rl)
|
|
2713
3243
|
return [x for x in self.glob(*rl) if self.path.isdir(x)]
|
|
2714
3244
|
|
|
2715
3245
|
@_kw2spawn
|
|
2716
3246
|
def gzip(self, *args, **kw):
|
|
2717
3247
|
"""Simple gzip compression of a file."""
|
|
2718
|
-
cmd = [
|
|
3248
|
+
cmd = ["gzip", "-vf", args[0]]
|
|
2719
3249
|
cmd.extend(args[1:])
|
|
2720
3250
|
return self.spawn(cmd, **kw)
|
|
2721
3251
|
|
|
2722
3252
|
@_kw2spawn
|
|
2723
3253
|
def gunzip(self, *args, **kw):
|
|
2724
3254
|
"""Simple gunzip of a gzip-compressed file."""
|
|
2725
|
-
cmd = [
|
|
3255
|
+
cmd = ["gunzip", args[0]]
|
|
2726
3256
|
cmd.extend(args[1:])
|
|
2727
3257
|
return self.spawn(cmd, **kw)
|
|
2728
3258
|
|
|
@@ -2734,21 +3264,21 @@ class OSExtended(System):
|
|
|
2734
3264
|
"""Build a proper string sequence of tar options."""
|
|
2735
3265
|
zopt = set(opts)
|
|
2736
3266
|
if verbose:
|
|
2737
|
-
zopt.add(
|
|
3267
|
+
zopt.add("v")
|
|
2738
3268
|
else:
|
|
2739
|
-
zopt.discard(
|
|
3269
|
+
zopt.discard("v")
|
|
2740
3270
|
if autocompress:
|
|
2741
|
-
if tarfile.endswith(
|
|
3271
|
+
if tarfile.endswith("gz"):
|
|
2742
3272
|
# includes the conventional "*.tgz"
|
|
2743
|
-
zopt.add(
|
|
3273
|
+
zopt.add("z")
|
|
2744
3274
|
else:
|
|
2745
|
-
zopt.discard(
|
|
2746
|
-
if tarfile.endswith(
|
|
3275
|
+
zopt.discard("z")
|
|
3276
|
+
if tarfile.endswith("bz") or tarfile.endswith("bz2"):
|
|
2747
3277
|
# includes the conventional "*.tbz"
|
|
2748
|
-
zopt.add(
|
|
3278
|
+
zopt.add("j")
|
|
2749
3279
|
else:
|
|
2750
|
-
zopt.discard(
|
|
2751
|
-
return
|
|
3280
|
+
zopt.discard("j")
|
|
3281
|
+
return "".join(zopt)
|
|
2752
3282
|
|
|
2753
3283
|
@_kw2spawn
|
|
2754
3284
|
def tar(self, *args, **kw):
|
|
@@ -2756,8 +3286,13 @@ class OSExtended(System):
|
|
|
2756
3286
|
|
|
2757
3287
|
:example: ``self.tar('destination.tar', 'directory1', 'directory2')``
|
|
2758
3288
|
"""
|
|
2759
|
-
opts = self.taropts(
|
|
2760
|
-
|
|
3289
|
+
opts = self.taropts(
|
|
3290
|
+
args[0],
|
|
3291
|
+
"cf",
|
|
3292
|
+
kw.pop("verbose", True),
|
|
3293
|
+
kw.pop("autocompress", True),
|
|
3294
|
+
)
|
|
3295
|
+
cmd = ["tar", opts, args[0]]
|
|
2761
3296
|
cmd.extend(self.glob(*args[1:]))
|
|
2762
3297
|
return self.spawn(cmd, **kw)
|
|
2763
3298
|
|
|
@@ -2768,8 +3303,13 @@ class OSExtended(System):
|
|
|
2768
3303
|
:example: ``self.untar('source.tar')``
|
|
2769
3304
|
:example: ``self.untar('source.tar', 'to_untar1', 'to_untar2')``
|
|
2770
3305
|
"""
|
|
2771
|
-
opts = self.taropts(
|
|
2772
|
-
|
|
3306
|
+
opts = self.taropts(
|
|
3307
|
+
args[0],
|
|
3308
|
+
"xf",
|
|
3309
|
+
kw.pop("verbose", True),
|
|
3310
|
+
kw.pop("autocompress", True),
|
|
3311
|
+
)
|
|
3312
|
+
cmd = ["tar", opts, args[0]]
|
|
2773
3313
|
cmd.extend(args[1:])
|
|
2774
3314
|
return self.spawn(cmd, **kw)
|
|
2775
3315
|
|
|
@@ -2784,56 +3324,69 @@ class OSExtended(System):
|
|
|
2784
3324
|
This is done in a relatively safe way since it is checked that no existing
|
|
2785
3325
|
files/directories are overwritten.
|
|
2786
3326
|
"""
|
|
2787
|
-
uniquelevel_ignore = kw.pop(
|
|
3327
|
+
uniquelevel_ignore = kw.pop("uniquelevel_ignore", False)
|
|
2788
3328
|
fullsource = self.path.realpath(source)
|
|
2789
3329
|
self.mkdir(destination)
|
|
2790
|
-
loctmp = tempfile.mkdtemp(prefix=
|
|
3330
|
+
loctmp = tempfile.mkdtemp(prefix="untar_", dir=destination)
|
|
2791
3331
|
with self.cdcontext(loctmp, clean_onexit=True):
|
|
2792
|
-
output_setting = kw.pop(
|
|
3332
|
+
output_setting = kw.pop("output", True)
|
|
2793
3333
|
output_txt = self.untar(fullsource, output=output_setting, **kw)
|
|
2794
3334
|
if output_setting and output_txt:
|
|
2795
|
-
logger.info(
|
|
2796
|
-
unpacked = self.glob(
|
|
2797
|
-
unpacked_prefix =
|
|
3335
|
+
logger.info("Untar command output:\n%s", "\n".join(output_txt))
|
|
3336
|
+
unpacked = self.glob("*")
|
|
3337
|
+
unpacked_prefix = "."
|
|
2798
3338
|
# If requested, ignore the first level of directory
|
|
2799
|
-
if (
|
|
2800
|
-
|
|
3339
|
+
if (
|
|
3340
|
+
uniquelevel_ignore
|
|
3341
|
+
and len(unpacked) == 1
|
|
3342
|
+
and self.path.isdir(self.path.join(unpacked[0]))
|
|
3343
|
+
):
|
|
2801
3344
|
unpacked_prefix = unpacked[0]
|
|
2802
|
-
logger.info(
|
|
3345
|
+
logger.info(
|
|
3346
|
+
"Moving contents one level up: %s", unpacked_prefix
|
|
3347
|
+
)
|
|
2803
3348
|
with self.cdcontext(unpacked_prefix):
|
|
2804
|
-
unpacked = self.glob(
|
|
3349
|
+
unpacked = self.glob("*")
|
|
2805
3350
|
for untaritem in unpacked:
|
|
2806
|
-
itemtarget = self.path.join(
|
|
3351
|
+
itemtarget = self.path.join(
|
|
3352
|
+
"..", self.path.basename(untaritem)
|
|
3353
|
+
)
|
|
2807
3354
|
if self.path.exists(itemtarget):
|
|
2808
|
-
logger.error(
|
|
3355
|
+
logger.error(
|
|
3356
|
+
"Some previous item exists before untar [%s]",
|
|
3357
|
+
untaritem,
|
|
3358
|
+
)
|
|
2809
3359
|
else:
|
|
2810
|
-
self.mv(
|
|
2811
|
-
|
|
3360
|
+
self.mv(
|
|
3361
|
+
self.path.join(unpacked_prefix, untaritem), itemtarget
|
|
3362
|
+
)
|
|
2812
3363
|
return unpacked
|
|
2813
3364
|
|
|
2814
3365
|
def is_tarname(self, objname):
|
|
2815
3366
|
"""Check if a ``objname`` is a string with ``.tar`` suffix."""
|
|
2816
|
-
return isinstance(objname, str) and (
|
|
2817
|
-
|
|
2818
|
-
|
|
2819
|
-
|
|
2820
|
-
|
|
3367
|
+
return isinstance(objname, str) and (
|
|
3368
|
+
objname.endswith(".tar")
|
|
3369
|
+
or objname.endswith(".tar.gz")
|
|
3370
|
+
or objname.endswith(".tgz")
|
|
3371
|
+
or objname.endswith(".tar.bz2")
|
|
3372
|
+
or objname.endswith(".tbz")
|
|
3373
|
+
)
|
|
2821
3374
|
|
|
2822
3375
|
def tarname_radix(self, objname):
|
|
2823
3376
|
"""Remove any ``.tar`` specific suffix."""
|
|
2824
3377
|
if not self.is_tarname(objname):
|
|
2825
3378
|
return objname
|
|
2826
3379
|
radix = self.path.splitext(objname)[0]
|
|
2827
|
-
if radix.endswith(
|
|
3380
|
+
if radix.endswith(".tar"):
|
|
2828
3381
|
radix = radix[:-4]
|
|
2829
3382
|
return radix
|
|
2830
3383
|
|
|
2831
3384
|
def tarname_splitext(self, objname):
|
|
2832
3385
|
"""Like os.path.splitext, but for tar names (e.g. might return ``.tar.gz``)."""
|
|
2833
3386
|
if not self.is_tarname(objname):
|
|
2834
|
-
return (objname,
|
|
3387
|
+
return (objname, "")
|
|
2835
3388
|
radix = self.tarname_radix(objname)
|
|
2836
|
-
ext = objname.replace(radix,
|
|
3389
|
+
ext = objname.replace(radix, "")
|
|
2837
3390
|
return (radix, ext)
|
|
2838
3391
|
|
|
2839
3392
|
@fmtshcmd
|
|
@@ -2852,12 +3405,14 @@ class OSExtended(System):
|
|
|
2852
3405
|
(either a file descriptor or a filename).
|
|
2853
3406
|
"""
|
|
2854
3407
|
rc = None
|
|
2855
|
-
if hasattr(destination,
|
|
3408
|
+
if hasattr(destination, "write"):
|
|
2856
3409
|
rc = gateway.dump(obj, destination, **opts)
|
|
2857
3410
|
else:
|
|
2858
3411
|
if self.filecocoon(destination):
|
|
2859
|
-
with open(
|
|
2860
|
-
|
|
3412
|
+
with open(
|
|
3413
|
+
self.path.expanduser(destination),
|
|
3414
|
+
"w" + ("b" if bytesdump else ""),
|
|
3415
|
+
) as fd:
|
|
2861
3416
|
rc = gateway.dump(obj, fd, **opts)
|
|
2862
3417
|
return rc
|
|
2863
3418
|
|
|
@@ -2866,7 +3421,9 @@ class OSExtended(System):
|
|
|
2866
3421
|
Dump a pickled representation of specified **obj** in file **destination**,
|
|
2867
3422
|
(either a file descriptor or a filename).
|
|
2868
3423
|
"""
|
|
2869
|
-
return self.blind_dump(
|
|
3424
|
+
return self.blind_dump(
|
|
3425
|
+
pickle, obj, destination, bytesdump=True, **opts
|
|
3426
|
+
)
|
|
2870
3427
|
|
|
2871
3428
|
def json_dump(self, obj, destination, **opts):
|
|
2872
3429
|
"""
|
|
@@ -2880,11 +3437,12 @@ class OSExtended(System):
|
|
|
2880
3437
|
Use **gateway** for a blind load the representation stored in file **source**,
|
|
2881
3438
|
(either a file descriptor or a filename).
|
|
2882
3439
|
"""
|
|
2883
|
-
if hasattr(source,
|
|
3440
|
+
if hasattr(source, "read"):
|
|
2884
3441
|
obj = gateway.load(source)
|
|
2885
3442
|
else:
|
|
2886
|
-
with open(
|
|
2887
|
-
|
|
3443
|
+
with open(
|
|
3444
|
+
self.path.expanduser(source), "r" + ("b" if bytesload else "")
|
|
3445
|
+
) as fd:
|
|
2888
3446
|
obj = gateway.load(fd)
|
|
2889
3447
|
return obj
|
|
2890
3448
|
|
|
@@ -2909,13 +3467,17 @@ class OSExtended(System):
|
|
|
2909
3467
|
def utlines(self, *args):
|
|
2910
3468
|
"""Return number of significant code or configuration lines in specified directories."""
|
|
2911
3469
|
lookfiles = [
|
|
2912
|
-
x
|
|
2913
|
-
|
|
3470
|
+
x
|
|
3471
|
+
for x in self.ffind(*args)
|
|
3472
|
+
if self.path.splitext[1] in [".py", ".ini", ".tpl", ".rst"]
|
|
2914
3473
|
]
|
|
2915
|
-
return len(
|
|
2916
|
-
|
|
2917
|
-
|
|
2918
|
-
|
|
3474
|
+
return len(
|
|
3475
|
+
[
|
|
3476
|
+
x
|
|
3477
|
+
for x in self.cat(*lookfiles)
|
|
3478
|
+
if re.search(r"\S", x) and re.search(r"[^\'\"\)\],\s]", x)
|
|
3479
|
+
]
|
|
3480
|
+
)
|
|
2919
3481
|
|
|
2920
3482
|
def _signal_intercept_init(self):
|
|
2921
3483
|
"""Initialise the signal handler object (but do not activate it)."""
|
|
@@ -2935,7 +3497,9 @@ class OSExtended(System):
|
|
|
2935
3497
|
"""
|
|
2936
3498
|
self._sighandler.deactivate()
|
|
2937
3499
|
|
|
2938
|
-
_LDD_REGEX = re.compile(
|
|
3500
|
+
_LDD_REGEX = re.compile(
|
|
3501
|
+
r"^\s*([^\s]+)\s+=>\s*(?:([^\s]+)\s+\(0x.+\)|not found)$"
|
|
3502
|
+
)
|
|
2939
3503
|
|
|
2940
3504
|
def ldd(self, filename):
|
|
2941
3505
|
"""Call ldd on **filename**.
|
|
@@ -2943,14 +3507,14 @@ class OSExtended(System):
|
|
|
2943
3507
|
Return the mapping between the library name and its physical path.
|
|
2944
3508
|
"""
|
|
2945
3509
|
if self.path.isfile(filename):
|
|
2946
|
-
ldd_out = self.spawn((
|
|
3510
|
+
ldd_out = self.spawn(("ldd", filename))
|
|
2947
3511
|
libs = dict()
|
|
2948
3512
|
for ldd_match in [self._LDD_REGEX.match(l) for l in ldd_out]:
|
|
2949
3513
|
if ldd_match is not None:
|
|
2950
3514
|
libs[ldd_match.group(1)] = ldd_match.group(2) or None
|
|
2951
3515
|
return libs
|
|
2952
3516
|
else:
|
|
2953
|
-
raise ValueError(
|
|
3517
|
+
raise ValueError("{} is not a regular file".format(filename))
|
|
2954
3518
|
|
|
2955
3519
|
def generic_compress(self, pipelinedesc, source, destination=None):
|
|
2956
3520
|
"""Compress a file using the :class:`CompressionPipeline` class.
|
|
@@ -2964,7 +3528,9 @@ class OSExtended(System):
|
|
|
2964
3528
|
if isinstance(source, str):
|
|
2965
3529
|
destination = source + cp.suffix
|
|
2966
3530
|
else:
|
|
2967
|
-
raise ValueError(
|
|
3531
|
+
raise ValueError(
|
|
3532
|
+
"If destination is omitted, source must be a filename."
|
|
3533
|
+
)
|
|
2968
3534
|
return cp.compress2file(source, destination)
|
|
2969
3535
|
|
|
2970
3536
|
def generic_uncompress(self, pipelinedesc, source, destination=None):
|
|
@@ -2978,11 +3544,17 @@ class OSExtended(System):
|
|
|
2978
3544
|
if destination is None:
|
|
2979
3545
|
if isinstance(source, str):
|
|
2980
3546
|
if source.endswith(cp.suffix):
|
|
2981
|
-
destination = source[
|
|
3547
|
+
destination = source[: -len(cp.suffix)]
|
|
2982
3548
|
else:
|
|
2983
|
-
raise ValueError(
|
|
3549
|
+
raise ValueError(
|
|
3550
|
+
"Source do not exhibit the appropriate suffix ({:s})".format(
|
|
3551
|
+
cp.suffix
|
|
3552
|
+
)
|
|
3553
|
+
)
|
|
2984
3554
|
else:
|
|
2985
|
-
raise ValueError(
|
|
3555
|
+
raise ValueError(
|
|
3556
|
+
"If destination is omitted, source must be a filename."
|
|
3557
|
+
)
|
|
2986
3558
|
return cp.file2uncompress(source, destination)
|
|
2987
3559
|
|
|
2988
3560
|
def find_mount_point(self, path):
|
|
@@ -2993,7 +3565,7 @@ class OSExtended(System):
|
|
|
2993
3565
|
:rtype: str
|
|
2994
3566
|
"""
|
|
2995
3567
|
if not self._os.path.exists(path):
|
|
2996
|
-
logger.warning(
|
|
3568
|
+
logger.warning("Path does not exist: <%s>", path)
|
|
2997
3569
|
|
|
2998
3570
|
path = self._os.path.abspath(path)
|
|
2999
3571
|
while not self._os.path.ismount(path):
|
|
@@ -3013,7 +3585,9 @@ class OSExtended(System):
|
|
|
3013
3585
|
"""
|
|
3014
3586
|
rc = None
|
|
3015
3587
|
t0 = time.time()
|
|
3016
|
-
while rc is None or (
|
|
3588
|
+
while rc is None or (
|
|
3589
|
+
not rc and blocking and time.time() - t0 < timeout
|
|
3590
|
+
):
|
|
3017
3591
|
if rc is not None:
|
|
3018
3592
|
self.sleep(sleeptime)
|
|
3019
3593
|
try:
|
|
@@ -3021,7 +3595,7 @@ class OSExtended(System):
|
|
|
3021
3595
|
# since we need to get an error if the target directory already
|
|
3022
3596
|
# exists
|
|
3023
3597
|
self._os.mkdir(ldir)
|
|
3024
|
-
except FileExistsError
|
|
3598
|
+
except FileExistsError:
|
|
3025
3599
|
rc = False
|
|
3026
3600
|
else:
|
|
3027
3601
|
rc = True
|
|
@@ -3038,8 +3612,14 @@ class OSExtended(System):
|
|
|
3038
3612
|
logger.warning("'%s' did not exists... that's odd", ldir)
|
|
3039
3613
|
|
|
3040
3614
|
@contextlib.contextmanager
|
|
3041
|
-
def lockdir_context(
|
|
3042
|
-
|
|
3615
|
+
def lockdir_context(
|
|
3616
|
+
self,
|
|
3617
|
+
ldir,
|
|
3618
|
+
sloppy=False,
|
|
3619
|
+
timeout=120,
|
|
3620
|
+
sleeptime_min=0.1,
|
|
3621
|
+
sleeptime_max=0.3,
|
|
3622
|
+
):
|
|
3043
3623
|
"""Try to acquire a lock directory and after that remove it.
|
|
3044
3624
|
|
|
3045
3625
|
:param bool sloppy: If the lock can be acquired after *timeout* second, go on anyway
|
|
@@ -3049,14 +3629,20 @@ class OSExtended(System):
|
|
|
3049
3629
|
:param float sleeptime_max: When blocking, wait at most **sleeptime_max** seconds
|
|
3050
3630
|
between to attempts to acquire the lock.
|
|
3051
3631
|
"""
|
|
3052
|
-
sleeptime =
|
|
3632
|
+
sleeptime = (
|
|
3633
|
+
sleeptime_min + (sleeptime_max - sleeptime_min) * random.random()
|
|
3634
|
+
)
|
|
3053
3635
|
self.filecocoon(ldir)
|
|
3054
|
-
rc = self._lockdir_create(
|
|
3636
|
+
rc = self._lockdir_create(
|
|
3637
|
+
ldir, blocking=True, timeout=timeout, sleeptime=sleeptime
|
|
3638
|
+
)
|
|
3055
3639
|
try:
|
|
3056
3640
|
if not rc:
|
|
3057
|
-
msg = "Could not acquire lockdir < {:s} >. Already exists.".format(
|
|
3641
|
+
msg = "Could not acquire lockdir < {:s} >. Already exists.".format(
|
|
3642
|
+
ldir
|
|
3643
|
+
)
|
|
3058
3644
|
if sloppy:
|
|
3059
|
-
logger.warning(msg +
|
|
3645
|
+
logger.warning(msg + ".. but going on.")
|
|
3060
3646
|
else:
|
|
3061
3647
|
raise OSError(msg)
|
|
3062
3648
|
yield
|
|
@@ -3070,9 +3656,11 @@ class OSExtended(System):
|
|
|
3070
3656
|
if self.glove is not None:
|
|
3071
3657
|
myglove = self.glove
|
|
3072
3658
|
rcdir = myglove.configrc
|
|
3073
|
-
lockdir = self.path.join(
|
|
3074
|
-
|
|
3075
|
-
|
|
3659
|
+
lockdir = self.path.join(
|
|
3660
|
+
rcdir,
|
|
3661
|
+
"appwide_locks",
|
|
3662
|
+
"{0.vapp:s}-{0.vconf:s}".format(myglove),
|
|
3663
|
+
)
|
|
3076
3664
|
self.mkdir(lockdir)
|
|
3077
3665
|
return lockdir
|
|
3078
3666
|
else:
|
|
@@ -3097,10 +3685,9 @@ class OSExtended(System):
|
|
|
3097
3685
|
attempts to acquire the lock.
|
|
3098
3686
|
"""
|
|
3099
3687
|
ldir = self._appwide_lockdir_path(label)
|
|
3100
|
-
return self._lockdir_create(
|
|
3101
|
-
|
|
3102
|
-
|
|
3103
|
-
sleeptime=sleeptime)
|
|
3688
|
+
return self._lockdir_create(
|
|
3689
|
+
ldir, blocking=blocking, timeout=timeout, sleeptime=sleeptime
|
|
3690
|
+
)
|
|
3104
3691
|
|
|
3105
3692
|
def appwide_unlock(self, label):
|
|
3106
3693
|
"""Pseudo-lock mechanism based on atomic directory creation: release lock.
|
|
@@ -3124,9 +3711,8 @@ class Python34:
|
|
|
3124
3711
|
"""
|
|
3125
3712
|
|
|
3126
3713
|
# Optional, netcdf comparison tool
|
|
3127
|
-
b_netcdf_checker = ExternalCodeImportChecker(
|
|
3128
|
-
|
|
3129
|
-
from bronx.datagrip import netcdf as b_netcdf
|
|
3714
|
+
b_netcdf_checker = ExternalCodeImportChecker("netdcf")
|
|
3715
|
+
from bronx.datagrip import netcdf as b_netcdf
|
|
3130
3716
|
|
|
3131
3717
|
if b_netcdf_checker.is_available():
|
|
3132
3718
|
# Unfortunately, the netCDF4 package seems to leak memory,
|
|
@@ -3136,25 +3722,28 @@ class Python34:
|
|
|
3136
3722
|
"""Function started by the subprocess."""
|
|
3137
3723
|
outcome.value = int(b_netcdf.netcdf_file_diff(nc1, nc2))
|
|
3138
3724
|
|
|
3139
|
-
rc = multiprocessing.Value(
|
|
3140
|
-
p = multiprocessing.Process(
|
|
3141
|
-
|
|
3725
|
+
rc = multiprocessing.Value("i", 0)
|
|
3726
|
+
p = multiprocessing.Process(
|
|
3727
|
+
target=_compare_function, args=(netcdf1, netcdf2, rc)
|
|
3728
|
+
)
|
|
3142
3729
|
p.start()
|
|
3143
3730
|
p.join()
|
|
3144
3731
|
return bool(rc.value)
|
|
3145
3732
|
else:
|
|
3146
|
-
logger.error(
|
|
3147
|
-
|
|
3733
|
+
logger.error(
|
|
3734
|
+
"Unable to load the 'bronx.datagrip.netcdf' package. "
|
|
3735
|
+
+ "The netcdf library and/or 'netCDF4' python package are probably missing."
|
|
3736
|
+
)
|
|
3148
3737
|
return False
|
|
3149
3738
|
|
|
3150
3739
|
# Let's make this method compatible with fmtshcmd...
|
|
3151
3740
|
netcdf_diff.func_extern = True
|
|
3152
3741
|
|
|
3153
3742
|
|
|
3154
|
-
_python34_fp = footprints.Footprint(
|
|
3155
|
-
|
|
3156
|
-
|
|
3157
|
-
|
|
3743
|
+
_python34_fp = footprints.Footprint(
|
|
3744
|
+
info="An abstract footprint to be used with the Python34 Mixin",
|
|
3745
|
+
only=dict(after_python=PythonSimplifiedVersion("3.4.0")),
|
|
3746
|
+
)
|
|
3158
3747
|
|
|
3159
3748
|
|
|
3160
3749
|
class Garbage(OSExtended):
|
|
@@ -3166,20 +3755,18 @@ class Garbage(OSExtended):
|
|
|
3166
3755
|
|
|
3167
3756
|
_abstract = True
|
|
3168
3757
|
_footprint = dict(
|
|
3169
|
-
info
|
|
3170
|
-
attr
|
|
3171
|
-
sysname
|
|
3172
|
-
outcast
|
|
3758
|
+
info="Garbage base system",
|
|
3759
|
+
attr=dict(
|
|
3760
|
+
sysname=dict(
|
|
3761
|
+
outcast=["Linux", "Darwin", "UnitTestLinux", "UnitTestable"]
|
|
3173
3762
|
)
|
|
3174
3763
|
),
|
|
3175
|
-
priority
|
|
3176
|
-
level = footprints.priorities.top.DEFAULT
|
|
3177
|
-
)
|
|
3764
|
+
priority=dict(level=footprints.priorities.top.DEFAULT),
|
|
3178
3765
|
)
|
|
3179
3766
|
|
|
3180
3767
|
def __init__(self, *args, **kw):
|
|
3181
3768
|
"""Gateway to parent method after debug logging."""
|
|
3182
|
-
logger.debug(
|
|
3769
|
+
logger.debug("Garbage system init %s", self.__class__)
|
|
3183
3770
|
super().__init__(*args, **kw)
|
|
3184
3771
|
|
|
3185
3772
|
|
|
@@ -3188,7 +3775,7 @@ class Garbage34p(Garbage, Python34):
|
|
|
3188
3775
|
|
|
3189
3776
|
_footprint = [
|
|
3190
3777
|
_python34_fp,
|
|
3191
|
-
dict(info
|
|
3778
|
+
dict(info="Garbage base system withh a blazing Python version"),
|
|
3192
3779
|
]
|
|
3193
3780
|
|
|
3194
3781
|
|
|
@@ -3197,12 +3784,8 @@ class Linux(OSExtended):
|
|
|
3197
3784
|
|
|
3198
3785
|
_abstract = True
|
|
3199
3786
|
_footprint = dict(
|
|
3200
|
-
info
|
|
3201
|
-
attr
|
|
3202
|
-
sysname = dict(
|
|
3203
|
-
values = ['Linux']
|
|
3204
|
-
)
|
|
3205
|
-
)
|
|
3787
|
+
info="Abstract Linux base system",
|
|
3788
|
+
attr=dict(sysname=dict(values=["Linux"])),
|
|
3206
3789
|
)
|
|
3207
3790
|
|
|
3208
3791
|
def __init__(self, *args, **kw):
|
|
@@ -3212,78 +3795,86 @@ class Linux(OSExtended):
|
|
|
3212
3795
|
|
|
3213
3796
|
* **psopts** - as default option for the ps command (default: ``-w -f -a``).
|
|
3214
3797
|
"""
|
|
3215
|
-
logger.debug(
|
|
3216
|
-
self._psopts = kw.pop(
|
|
3798
|
+
logger.debug("Linux system init %s", self.__class__)
|
|
3799
|
+
self._psopts = kw.pop("psopts", ["-w", "-f", "-a"])
|
|
3217
3800
|
super().__init__(*args, **kw)
|
|
3218
|
-
self.__dict__[
|
|
3801
|
+
self.__dict__["_cpusinfo"] = LinuxCpusInfo()
|
|
3219
3802
|
try:
|
|
3220
|
-
self.__dict__[
|
|
3803
|
+
self.__dict__["_numainfo"] = LibNumaNodesInfo()
|
|
3221
3804
|
except (OSError, NotImplementedError):
|
|
3222
3805
|
# On very few Linux systems, libnuma is not available...
|
|
3223
3806
|
pass
|
|
3224
|
-
self.__dict__[
|
|
3225
|
-
self.__dict__[
|
|
3807
|
+
self.__dict__["_memoryinfo"] = LinuxMemInfo()
|
|
3808
|
+
self.__dict__["_netstatsinfo"] = LinuxNetstats()
|
|
3226
3809
|
|
|
3227
3810
|
@property
|
|
3228
3811
|
def realkind(self):
|
|
3229
|
-
return
|
|
3812
|
+
return "linux"
|
|
3230
3813
|
|
|
3231
|
-
def cpus_ids_per_blocks(self, blocksize=1, topology=
|
|
3814
|
+
def cpus_ids_per_blocks(self, blocksize=1, topology="raw", hexmask=False):
|
|
3232
3815
|
"""Get the list of CPUs IDs for nicely ordered for subsequent binding.
|
|
3233
3816
|
|
|
3234
3817
|
:param int blocksize: the number of thread consumed by one task
|
|
3235
3818
|
:param str topology: The task distribution scheme
|
|
3236
3819
|
:param bool hexmask: Return a list of CPU masks in hexadecimal
|
|
3237
3820
|
"""
|
|
3238
|
-
if topology.startswith(
|
|
3239
|
-
if topology.endswith(
|
|
3821
|
+
if topology.startswith("numa"):
|
|
3822
|
+
if topology.endswith("_discardsmt"):
|
|
3240
3823
|
topology = topology[:-11]
|
|
3241
3824
|
smtlayout = None
|
|
3242
3825
|
else:
|
|
3243
3826
|
smtlayout = self.cpus_info.physical_cores_smtthreads
|
|
3244
3827
|
try:
|
|
3245
|
-
cpulist = getattr(self.numa_info,
|
|
3246
|
-
|
|
3247
|
-
|
|
3828
|
+
cpulist = getattr(self.numa_info, topology + "_cpulist")(
|
|
3829
|
+
blocksize, smtlayout=smtlayout
|
|
3830
|
+
)
|
|
3248
3831
|
except AttributeError:
|
|
3249
|
-
raise ValueError(
|
|
3832
|
+
raise ValueError("Unknown topology ({:s}).".format(topology))
|
|
3250
3833
|
else:
|
|
3251
3834
|
try:
|
|
3252
|
-
cpulist = getattr(self.cpus_info, topology +
|
|
3835
|
+
cpulist = getattr(self.cpus_info, topology + "_cpulist")(
|
|
3836
|
+
blocksize
|
|
3837
|
+
)
|
|
3253
3838
|
except AttributeError:
|
|
3254
|
-
raise ValueError(
|
|
3839
|
+
raise ValueError("Unknown topology ({:s}).".format(topology))
|
|
3255
3840
|
cpulist = list(cpulist)
|
|
3256
|
-
cpulist = [
|
|
3257
|
-
|
|
3258
|
-
|
|
3841
|
+
cpulist = [
|
|
3842
|
+
[cpulist[(taskid * blocksize + i)] for i in range(blocksize)]
|
|
3843
|
+
for taskid in range(len(cpulist) // blocksize)
|
|
3844
|
+
]
|
|
3259
3845
|
if hexmask:
|
|
3260
3846
|
cpulist = [hex(sum([1 << i for i in item])) for item in cpulist]
|
|
3261
3847
|
return cpulist
|
|
3262
3848
|
|
|
3263
|
-
def cpus_ids_dispenser(self, topology=
|
|
3849
|
+
def cpus_ids_dispenser(self, topology="raw"):
|
|
3264
3850
|
"""Get a dispenser of CPUs IDs for nicely ordered for subsequent binding.
|
|
3265
3851
|
|
|
3266
3852
|
:param str topology: The task distribution scheme
|
|
3267
3853
|
"""
|
|
3268
|
-
if topology.startswith(
|
|
3269
|
-
if topology.endswith(
|
|
3854
|
+
if topology.startswith("numa"):
|
|
3855
|
+
if topology.endswith("_discardsmt"):
|
|
3270
3856
|
topology = topology[:-11]
|
|
3271
3857
|
smtlayout = None
|
|
3272
3858
|
else:
|
|
3273
3859
|
smtlayout = self.cpus_info.physical_cores_smtthreads
|
|
3274
3860
|
try:
|
|
3275
|
-
cpudisp = getattr(self.numa_info,
|
|
3276
|
-
|
|
3861
|
+
cpudisp = getattr(self.numa_info, topology + "_cpu_dispenser")(
|
|
3862
|
+
smtlayout=smtlayout
|
|
3863
|
+
)
|
|
3277
3864
|
except AttributeError:
|
|
3278
|
-
raise ValueError(
|
|
3865
|
+
raise ValueError("Unknown topology ({:s}).".format(topology))
|
|
3279
3866
|
else:
|
|
3280
3867
|
try:
|
|
3281
|
-
cpudisp = getattr(
|
|
3868
|
+
cpudisp = getattr(
|
|
3869
|
+
self.cpus_info, topology + "_cpu_dispenser"
|
|
3870
|
+
)()
|
|
3282
3871
|
except AttributeError:
|
|
3283
|
-
raise ValueError(
|
|
3872
|
+
raise ValueError("Unknown topology ({:s}).".format(topology))
|
|
3284
3873
|
return cpudisp
|
|
3285
3874
|
|
|
3286
|
-
def cpus_affinity_get(
|
|
3875
|
+
def cpus_affinity_get(
|
|
3876
|
+
self, taskid, blocksize=1, topology="socketpacked", method="taskset"
|
|
3877
|
+
):
|
|
3287
3878
|
"""Get the necessary command/environment to set the CPUs affinity.
|
|
3288
3879
|
|
|
3289
3880
|
:param int taskid: the task number
|
|
@@ -3293,25 +3884,29 @@ class Linux(OSExtended):
|
|
|
3293
3884
|
:return: A 3-elements tuple. (bool: BindingPossible,
|
|
3294
3885
|
list: Starting command prefix, dict: Environment update)
|
|
3295
3886
|
"""
|
|
3296
|
-
if method not in (
|
|
3297
|
-
raise ValueError(
|
|
3298
|
-
if method ==
|
|
3299
|
-
if not self.which(
|
|
3300
|
-
logger.warning(
|
|
3887
|
+
if method not in ("taskset", "gomp", "omp", "ompverbose"):
|
|
3888
|
+
raise ValueError("Unknown binding method ({:s}).".format(method))
|
|
3889
|
+
if method == "taskset":
|
|
3890
|
+
if not self.which("taskset"):
|
|
3891
|
+
logger.warning(
|
|
3892
|
+
"The taskset is program is missing. Going on without binding."
|
|
3893
|
+
)
|
|
3301
3894
|
return (False, list(), dict())
|
|
3302
|
-
cpulist = self.cpus_ids_per_blocks(
|
|
3895
|
+
cpulist = self.cpus_ids_per_blocks(
|
|
3896
|
+
blocksize=blocksize, topology=topology
|
|
3897
|
+
)
|
|
3303
3898
|
cpus = cpulist[taskid % len(cpulist)]
|
|
3304
3899
|
cmdl = list()
|
|
3305
3900
|
env = dict()
|
|
3306
|
-
if method ==
|
|
3307
|
-
cmdl += [
|
|
3308
|
-
elif method ==
|
|
3309
|
-
env[
|
|
3310
|
-
elif method.startswith(
|
|
3311
|
-
env[
|
|
3312
|
-
if method.endswith(
|
|
3313
|
-
env[
|
|
3314
|
-
env[
|
|
3901
|
+
if method == "taskset":
|
|
3902
|
+
cmdl += ["taskset", "--cpu-list", ",".join([str(c) for c in cpus])]
|
|
3903
|
+
elif method == "gomp":
|
|
3904
|
+
env["GOMP_CPU_AFFINITY"] = " ".join([str(c) for c in cpus])
|
|
3905
|
+
elif method.startswith("omp"):
|
|
3906
|
+
env["OMP_PLACES"] = ",".join(["{{{:d}}}".format(c) for c in cpus])
|
|
3907
|
+
if method.endswith("verbose"):
|
|
3908
|
+
env["OMP_DISPLAY_ENV"] = "TRUE"
|
|
3909
|
+
env["OMP_DISPLAY_AFFINITY"] = "TRUE"
|
|
3315
3910
|
return (True, cmdl, env)
|
|
3316
3911
|
|
|
3317
3912
|
|
|
@@ -3320,7 +3915,7 @@ class Linux34p(Linux, Python34):
|
|
|
3320
3915
|
|
|
3321
3916
|
_footprint = [
|
|
3322
3917
|
_python34_fp,
|
|
3323
|
-
dict(info
|
|
3918
|
+
dict(info="Linux based system with a blazing Python version"),
|
|
3324
3919
|
]
|
|
3325
3920
|
|
|
3326
3921
|
|
|
@@ -3328,26 +3923,24 @@ class LinuxDebug(Linux34p):
|
|
|
3328
3923
|
"""Special system class for crude debugging on Linux based systems."""
|
|
3329
3924
|
|
|
3330
3925
|
_footprint = dict(
|
|
3331
|
-
info
|
|
3332
|
-
attr
|
|
3333
|
-
version
|
|
3334
|
-
optional
|
|
3335
|
-
values
|
|
3336
|
-
remap
|
|
3337
|
-
dbug = 'debug'
|
|
3338
|
-
)
|
|
3926
|
+
info="Linux debug system",
|
|
3927
|
+
attr=dict(
|
|
3928
|
+
version=dict(
|
|
3929
|
+
optional=False,
|
|
3930
|
+
values=["dbug", "debug"],
|
|
3931
|
+
remap=dict(dbug="debug"),
|
|
3339
3932
|
)
|
|
3340
|
-
)
|
|
3933
|
+
),
|
|
3341
3934
|
)
|
|
3342
3935
|
|
|
3343
3936
|
def __init__(self, *args, **kw):
|
|
3344
3937
|
"""Gateway to parent method after debug logging."""
|
|
3345
|
-
logger.debug(
|
|
3938
|
+
logger.debug("LinuxDebug system init %s", self.__class__)
|
|
3346
3939
|
super().__init__(*args, **kw)
|
|
3347
3940
|
|
|
3348
3941
|
@property
|
|
3349
3942
|
def realkind(self):
|
|
3350
|
-
return
|
|
3943
|
+
return "linuxdebug"
|
|
3351
3944
|
|
|
3352
3945
|
|
|
3353
3946
|
class Macosx(OSExtended):
|
|
@@ -3355,15 +3948,11 @@ class Macosx(OSExtended):
|
|
|
3355
3948
|
|
|
3356
3949
|
_abstract = True
|
|
3357
3950
|
_footprint = dict(
|
|
3358
|
-
info
|
|
3359
|
-
attr
|
|
3360
|
-
sysname
|
|
3361
|
-
values = ['Darwin']
|
|
3362
|
-
),
|
|
3951
|
+
info="Apple Mac computer under Macosx",
|
|
3952
|
+
attr=dict(
|
|
3953
|
+
sysname=dict(values=["Darwin"]),
|
|
3363
3954
|
),
|
|
3364
|
-
priority
|
|
3365
|
-
level = footprints.priorities.top.TOOLBOX
|
|
3366
|
-
)
|
|
3955
|
+
priority=dict(level=footprints.priorities.top.TOOLBOX),
|
|
3367
3956
|
)
|
|
3368
3957
|
|
|
3369
3958
|
def __init__(self, *args, **kw):
|
|
@@ -3373,18 +3962,18 @@ class Macosx(OSExtended):
|
|
|
3373
3962
|
|
|
3374
3963
|
* **psopts** - as default option for the ps command (default: ``-w -f -a``).
|
|
3375
3964
|
"""
|
|
3376
|
-
logger.debug(
|
|
3377
|
-
self._psopts = kw.pop(
|
|
3965
|
+
logger.debug("Darwin system init %s", self.__class__)
|
|
3966
|
+
self._psopts = kw.pop("psopts", ["-w", "-f", "-a"])
|
|
3378
3967
|
super().__init__(*args, **kw)
|
|
3379
3968
|
|
|
3380
3969
|
@property
|
|
3381
3970
|
def realkind(self):
|
|
3382
|
-
return
|
|
3971
|
+
return "darwin"
|
|
3383
3972
|
|
|
3384
3973
|
@property
|
|
3385
3974
|
def default_syslog(self):
|
|
3386
3975
|
"""Address to use in logging.handler.SysLogHandler()."""
|
|
3387
|
-
return
|
|
3976
|
+
return "/var/run/syslog"
|
|
3388
3977
|
|
|
3389
3978
|
|
|
3390
3979
|
class Macosx34p(Macosx, Python34):
|
|
@@ -3392,5 +3981,7 @@ class Macosx34p(Macosx, Python34):
|
|
|
3392
3981
|
|
|
3393
3982
|
_footprint = [
|
|
3394
3983
|
_python34_fp,
|
|
3395
|
-
dict(
|
|
3984
|
+
dict(
|
|
3985
|
+
info="Apple Mac computer under Macosx with a blazing Python version"
|
|
3986
|
+
),
|
|
3396
3987
|
]
|