rucio-clients 32.8.6__py3-none-any.whl → 35.8.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of rucio-clients might be problematic. Click here for more details.
- rucio/__init__.py +0 -1
- rucio/alembicrevision.py +1 -2
- rucio/client/__init__.py +0 -1
- rucio/client/accountclient.py +45 -25
- rucio/client/accountlimitclient.py +37 -9
- rucio/client/baseclient.py +199 -154
- rucio/client/client.py +2 -3
- rucio/client/configclient.py +19 -6
- rucio/client/credentialclient.py +9 -4
- rucio/client/didclient.py +238 -63
- rucio/client/diracclient.py +13 -5
- rucio/client/downloadclient.py +162 -51
- rucio/client/exportclient.py +4 -4
- rucio/client/fileclient.py +3 -4
- rucio/client/importclient.py +4 -4
- rucio/client/lifetimeclient.py +21 -5
- rucio/client/lockclient.py +18 -8
- rucio/client/{metaclient.py → metaconventionsclient.py} +18 -15
- rucio/client/pingclient.py +0 -1
- rucio/client/replicaclient.py +15 -5
- rucio/client/requestclient.py +35 -19
- rucio/client/rseclient.py +133 -51
- rucio/client/ruleclient.py +29 -22
- rucio/client/scopeclient.py +8 -6
- rucio/client/subscriptionclient.py +47 -35
- rucio/client/touchclient.py +8 -4
- rucio/client/uploadclient.py +166 -82
- rucio/common/__init__.py +0 -1
- rucio/common/cache.py +4 -4
- rucio/common/config.py +52 -47
- rucio/common/constants.py +69 -2
- rucio/common/constraints.py +0 -1
- rucio/common/didtype.py +24 -22
- rucio/common/exception.py +281 -222
- rucio/common/extra.py +0 -1
- rucio/common/logging.py +54 -38
- rucio/common/pcache.py +122 -101
- rucio/common/plugins.py +153 -0
- rucio/common/policy.py +4 -4
- rucio/common/schema/__init__.py +17 -10
- rucio/common/schema/atlas.py +7 -5
- rucio/common/schema/belleii.py +7 -5
- rucio/common/schema/domatpc.py +7 -5
- rucio/common/schema/escape.py +7 -5
- rucio/common/schema/generic.py +8 -6
- rucio/common/schema/generic_multi_vo.py +7 -5
- rucio/common/schema/icecube.py +7 -5
- rucio/common/stomp_utils.py +0 -1
- rucio/common/stopwatch.py +0 -1
- rucio/common/test_rucio_server.py +2 -2
- rucio/common/types.py +262 -17
- rucio/common/utils.py +743 -451
- rucio/rse/__init__.py +3 -4
- rucio/rse/protocols/__init__.py +0 -1
- rucio/rse/protocols/bittorrent.py +184 -0
- rucio/rse/protocols/cache.py +1 -2
- rucio/rse/protocols/dummy.py +1 -2
- rucio/rse/protocols/gfal.py +12 -10
- rucio/rse/protocols/globus.py +7 -7
- rucio/rse/protocols/gsiftp.py +2 -3
- rucio/rse/protocols/http_cache.py +1 -2
- rucio/rse/protocols/mock.py +1 -2
- rucio/rse/protocols/ngarc.py +1 -2
- rucio/rse/protocols/posix.py +12 -13
- rucio/rse/protocols/protocol.py +116 -52
- rucio/rse/protocols/rclone.py +6 -7
- rucio/rse/protocols/rfio.py +4 -5
- rucio/rse/protocols/srm.py +9 -10
- rucio/rse/protocols/ssh.py +8 -9
- rucio/rse/protocols/storm.py +2 -3
- rucio/rse/protocols/webdav.py +17 -14
- rucio/rse/protocols/xrootd.py +23 -17
- rucio/rse/rsemanager.py +19 -7
- rucio/vcsversion.py +4 -4
- rucio/version.py +5 -13
- rucio_clients-35.8.0.data/data/requirements.client.txt +15 -0
- {rucio_clients-32.8.6.data → rucio_clients-35.8.0.data}/data/rucio_client/merge_rucio_configs.py +2 -5
- {rucio_clients-32.8.6.data → rucio_clients-35.8.0.data}/scripts/rucio +87 -85
- {rucio_clients-32.8.6.data → rucio_clients-35.8.0.data}/scripts/rucio-admin +45 -32
- {rucio_clients-32.8.6.dist-info → rucio_clients-35.8.0.dist-info}/METADATA +13 -13
- rucio_clients-35.8.0.dist-info/RECORD +88 -0
- {rucio_clients-32.8.6.dist-info → rucio_clients-35.8.0.dist-info}/WHEEL +1 -1
- {rucio_clients-32.8.6.dist-info → rucio_clients-35.8.0.dist-info}/licenses/AUTHORS.rst +3 -0
- rucio/common/schema/cms.py +0 -478
- rucio/common/schema/lsst.py +0 -423
- rucio_clients-32.8.6.data/data/requirements.txt +0 -55
- rucio_clients-32.8.6.dist-info/RECORD +0 -88
- {rucio_clients-32.8.6.data → rucio_clients-35.8.0.data}/data/etc/rse-accounts.cfg.template +0 -0
- {rucio_clients-32.8.6.data → rucio_clients-35.8.0.data}/data/etc/rucio.cfg.atlas.client.template +0 -0
- {rucio_clients-32.8.6.data → rucio_clients-35.8.0.data}/data/etc/rucio.cfg.template +0 -0
- {rucio_clients-32.8.6.dist-info → rucio_clients-35.8.0.dist-info}/licenses/LICENSE +0 -0
- {rucio_clients-32.8.6.dist-info → rucio_clients-35.8.0.dist-info}/top_level.txt +0 -0
rucio/rse/protocols/protocol.py
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
1
|
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
3
2
|
#
|
|
4
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -17,14 +16,16 @@
|
|
|
17
16
|
This module defines the base class for implementing a transfer protocol,
|
|
18
17
|
along with some of the default methods for LFN2PFN translations.
|
|
19
18
|
"""
|
|
20
|
-
|
|
21
19
|
import hashlib
|
|
22
20
|
import logging
|
|
21
|
+
from collections.abc import Callable, Mapping
|
|
23
22
|
from configparser import NoOptionError, NoSectionError
|
|
23
|
+
from typing import Any, Optional, TypeVar
|
|
24
24
|
from urllib.parse import urlparse
|
|
25
25
|
|
|
26
26
|
from rucio.common import config, exception
|
|
27
|
-
from rucio.common.
|
|
27
|
+
from rucio.common.constants import RseAttr
|
|
28
|
+
from rucio.common.plugins import PolicyPackageAlgorithms
|
|
28
29
|
from rucio.rse import rsemanager
|
|
29
30
|
|
|
30
31
|
if getattr(rsemanager, 'CLIENT_MODE', None):
|
|
@@ -36,13 +37,102 @@ if getattr(rsemanager, 'SERVER_MODE', None):
|
|
|
36
37
|
from rucio.core.rse import get_rse_vo
|
|
37
38
|
|
|
38
39
|
|
|
39
|
-
class
|
|
40
|
+
class RSEDeterministicScopeTranslation(PolicyPackageAlgorithms):
|
|
41
|
+
"""
|
|
42
|
+
Translates a pfn dictionary into a scope and name
|
|
43
|
+
"""
|
|
44
|
+
|
|
45
|
+
_algorithm_type = "pfn2lfn"
|
|
46
|
+
|
|
47
|
+
def __init__(self, vo: str = 'def'):
|
|
48
|
+
super().__init__()
|
|
49
|
+
|
|
50
|
+
self.register(RSEDeterministicScopeTranslation._default, "def")
|
|
51
|
+
self.register(RSEDeterministicScopeTranslation._atlas, "atlas")
|
|
52
|
+
|
|
53
|
+
logger = logging.getLogger(__name__)
|
|
54
|
+
|
|
55
|
+
try:
|
|
56
|
+
# Use the function defined in the policy package if it's configured so
|
|
57
|
+
algorithm_name = config.config_get('policy', self._algorithm_type)
|
|
58
|
+
except (NoOptionError, NoSectionError, RuntimeError):
|
|
59
|
+
# Don't use a function from the policy package. Use one defined in this class according to vo
|
|
60
|
+
logger.debug("PFN2LFN function will not be fetched from the policy package")
|
|
61
|
+
if super()._supports(self._algorithm_type, vo):
|
|
62
|
+
algorithm_name = vo
|
|
63
|
+
else:
|
|
64
|
+
algorithm_name = "def"
|
|
65
|
+
|
|
66
|
+
self.parser = self.get_parser(algorithm_name)
|
|
67
|
+
|
|
68
|
+
@classmethod
|
|
69
|
+
def get_parser(cls, algorithm_name: str) -> Callable[..., Any]:
|
|
70
|
+
return super()._get_one_algorithm(cls._algorithm_type, algorithm_name)
|
|
71
|
+
|
|
72
|
+
@classmethod
|
|
73
|
+
def register(
|
|
74
|
+
cls,
|
|
75
|
+
pfn2lfn_callable: Callable,
|
|
76
|
+
name: Optional[str] = None
|
|
77
|
+
) -> None:
|
|
78
|
+
"""
|
|
79
|
+
Provided a callable function, register it as one of the valid PFN2LFN algorithms.
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
:param pfn2lfn_callable: Callable function to use.
|
|
83
|
+
:param name: Algorithm name used for registration.
|
|
84
|
+
"""
|
|
85
|
+
if name is None:
|
|
86
|
+
name = pfn2lfn_callable.__name__
|
|
87
|
+
algorithm_dict = {name: pfn2lfn_callable}
|
|
88
|
+
super()._register(cls._algorithm_type, algorithm_dict)
|
|
89
|
+
|
|
90
|
+
@staticmethod
|
|
91
|
+
def _default(parsed_pfn: Mapping[str, str]) -> tuple[str, str]:
|
|
92
|
+
""" Translate pfn to name/scope pair
|
|
93
|
+
|
|
94
|
+
:param parsed_pfn: dictionary representing pfn containing:
|
|
95
|
+
- path: str,
|
|
96
|
+
- name: str
|
|
97
|
+
:return: tuple containing name, scope
|
|
98
|
+
"""
|
|
99
|
+
path = parsed_pfn['path']
|
|
100
|
+
scope = path.lstrip('/').split('/')[0]
|
|
101
|
+
name = parsed_pfn['name']
|
|
102
|
+
return name, scope
|
|
103
|
+
|
|
104
|
+
@staticmethod
|
|
105
|
+
def _atlas(parsed_pfn: Mapping[str, str]) -> tuple[str, str]:
|
|
106
|
+
""" Translate pfn to name/scope pair
|
|
107
|
+
|
|
108
|
+
:param parsed_pfn: dictionary representing pfn containing:
|
|
109
|
+
- path: str,
|
|
110
|
+
- name: str
|
|
111
|
+
:return: tuple containing name, scope
|
|
112
|
+
"""
|
|
113
|
+
path = parsed_pfn['path']
|
|
114
|
+
if path.startswith('/user') or path.startswith('/group'):
|
|
115
|
+
scope = '%s.%s' % (path.split('/')[1], path.split('/')[2])
|
|
116
|
+
name = parsed_pfn['name']
|
|
117
|
+
else:
|
|
118
|
+
name, scope = RSEDeterministicScopeTranslation._default(parsed_pfn)
|
|
119
|
+
|
|
120
|
+
return name, scope
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
RSEDeterministicScopeTranslation()
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
RSEDeterministicTranslationT = TypeVar('RSEDeterministicTranslationT', bound='RSEDeterministicTranslation')
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
class RSEDeterministicTranslation(PolicyPackageAlgorithms):
|
|
40
130
|
"""
|
|
41
131
|
Execute the logic for translating a LFN to a path.
|
|
42
132
|
"""
|
|
43
133
|
|
|
44
|
-
_LFN2PFN_ALGORITHMS = {}
|
|
45
134
|
_DEFAULT_LFN2PFN = "hash"
|
|
135
|
+
_algorithm_type = "lfn2pfn"
|
|
46
136
|
|
|
47
137
|
def __init__(self, rse=None, rse_attributes=None, protocol_attributes=None):
|
|
48
138
|
"""
|
|
@@ -53,10 +143,10 @@ class RSEDeterministicTranslation(object):
|
|
|
53
143
|
:param rse_attributes: A dictionary of RSE-specific attributes for use in the translation.
|
|
54
144
|
:param protocol_attributes: A dictionary of RSE/protocol-specific attributes.
|
|
55
145
|
"""
|
|
146
|
+
super().__init__()
|
|
56
147
|
self.rse = rse
|
|
57
148
|
self.rse_attributes = rse_attributes if rse_attributes else {}
|
|
58
149
|
self.protocol_attributes = protocol_attributes if protocol_attributes else {}
|
|
59
|
-
self.loaded_policy_modules = False
|
|
60
150
|
|
|
61
151
|
@classmethod
|
|
62
152
|
def supports(cls, name):
|
|
@@ -64,12 +154,12 @@ class RSEDeterministicTranslation(object):
|
|
|
64
154
|
Check to see if a specific algorithm is supported.
|
|
65
155
|
|
|
66
156
|
:param name: Name of the deterministic algorithm.
|
|
67
|
-
:returns: True if `name` is an algorithm supported by the translator class, False otherwise
|
|
157
|
+
:returns: True if `name` is an algorithm supported by the translator class, False otherwise
|
|
68
158
|
"""
|
|
69
|
-
return
|
|
159
|
+
return super()._supports(cls._algorithm_type, name)
|
|
70
160
|
|
|
71
|
-
@
|
|
72
|
-
def register(lfn2pfn_callable, name=None):
|
|
161
|
+
@classmethod
|
|
162
|
+
def register(cls, lfn2pfn_callable, name=None):
|
|
73
163
|
"""
|
|
74
164
|
Provided a callable function, register it as one of the valid LFN2PFN algorithms.
|
|
75
165
|
|
|
@@ -87,7 +177,8 @@ class RSEDeterministicTranslation(object):
|
|
|
87
177
|
"""
|
|
88
178
|
if name is None:
|
|
89
179
|
name = lfn2pfn_callable.__name__
|
|
90
|
-
|
|
180
|
+
algorithm_dict = {name: lfn2pfn_callable}
|
|
181
|
+
super()._register(cls._algorithm_type, algorithm_dict)
|
|
91
182
|
|
|
92
183
|
@staticmethod
|
|
93
184
|
def __hash(scope, name, rse, rse_attrs, protocol_attrs):
|
|
@@ -194,27 +285,6 @@ class RSEDeterministicTranslation(object):
|
|
|
194
285
|
|
|
195
286
|
return '%s/%s/%s/%s' % (scope[0:7], scope[4:len(scope)], name.split('-')[0] + "-" + name.split('-')[1], name)
|
|
196
287
|
|
|
197
|
-
@staticmethod
|
|
198
|
-
def __lsst(scope, name, rse, rse_attrs, protocol_attrs):
|
|
199
|
-
"""
|
|
200
|
-
LFN2PFN algorithm for Rubin-LSST in the ESCAPE project
|
|
201
|
-
|
|
202
|
-
Replace convention delimiter '__' by '/'
|
|
203
|
-
The Escape instance does use the 'generic' Rucio schema.
|
|
204
|
-
|
|
205
|
-
:param scope: Scope of the LFN (ignored)
|
|
206
|
-
:param name: File name of the LFN.
|
|
207
|
-
:param rse: RSE for PFN (ignored)
|
|
208
|
-
:param rse_attrs: RSE attributes for PFN (ignored)
|
|
209
|
-
:param protocol_attrs: RSE protocol attributes for PFN (ignored)
|
|
210
|
-
:returns: Path for use in the PFN generation.
|
|
211
|
-
"""
|
|
212
|
-
del scope
|
|
213
|
-
del rse
|
|
214
|
-
del rse_attrs
|
|
215
|
-
del protocol_attrs
|
|
216
|
-
return name.replace('__', '/')
|
|
217
|
-
|
|
218
288
|
@classmethod
|
|
219
289
|
def _module_init_(cls):
|
|
220
290
|
"""
|
|
@@ -225,7 +295,6 @@ class RSEDeterministicTranslation(object):
|
|
|
225
295
|
cls.register(cls.__ligo, "ligo")
|
|
226
296
|
cls.register(cls.__belleii, "belleii")
|
|
227
297
|
cls.register(cls.__xenon, "xenon")
|
|
228
|
-
cls.register(cls.__lsst, "lsst")
|
|
229
298
|
policy_module = None
|
|
230
299
|
try:
|
|
231
300
|
policy_module = config.config_get('policy', 'lfn2pfn_module')
|
|
@@ -247,22 +316,17 @@ class RSEDeterministicTranslation(object):
|
|
|
247
316
|
|
|
248
317
|
:returns: RSE specific URI of the physical file
|
|
249
318
|
"""
|
|
250
|
-
|
|
251
|
-
if getattr(rsemanager, 'SERVER_MODE', None) and not self.loaded_policy_modules:
|
|
252
|
-
register_policy_package_algorithms('lfn2pfn', RSEDeterministicTranslation._LFN2PFN_ALGORITHMS)
|
|
253
|
-
self.loaded_policy_modules = True
|
|
254
|
-
|
|
255
|
-
algorithm = self.rse_attributes.get('lfn2pfn_algorithm', 'default')
|
|
319
|
+
algorithm = self.rse_attributes.get(RseAttr.LFN2PFN_ALGORITHM, 'default')
|
|
256
320
|
if algorithm == 'default':
|
|
257
321
|
algorithm = RSEDeterministicTranslation._DEFAULT_LFN2PFN
|
|
258
|
-
algorithm_callable = RSEDeterministicTranslation.
|
|
322
|
+
algorithm_callable = super()._get_one_algorithm(RSEDeterministicTranslation._algorithm_type, algorithm)
|
|
259
323
|
return algorithm_callable(scope, name, self.rse, self.rse_attributes, self.protocol_attributes)
|
|
260
324
|
|
|
261
325
|
|
|
262
326
|
RSEDeterministicTranslation._module_init_() # pylint: disable=protected-access
|
|
263
327
|
|
|
264
328
|
|
|
265
|
-
class RSEProtocol
|
|
329
|
+
class RSEProtocol:
|
|
266
330
|
""" This class is virtual and acts as a base to inherit new protocols from. It further provides some common functionality which applies for the amjority of the protocols."""
|
|
267
331
|
|
|
268
332
|
def __init__(self, protocol_attr, rse_settings, logger=logging.log):
|
|
@@ -294,7 +358,7 @@ class RSEProtocol(object):
|
|
|
294
358
|
|
|
295
359
|
def lfns2pfns(self, lfns):
|
|
296
360
|
"""
|
|
297
|
-
|
|
361
|
+
Returns a fully qualified PFN for the file referred by path.
|
|
298
362
|
|
|
299
363
|
:param path: The path to the file.
|
|
300
364
|
|
|
@@ -335,7 +399,7 @@ class RSEProtocol(object):
|
|
|
335
399
|
return pfns
|
|
336
400
|
|
|
337
401
|
def __lfns2pfns_client(self, lfns):
|
|
338
|
-
""" Provides the path of a replica for non-deterministic sites. Will be assigned to get path by the __init__ method if
|
|
402
|
+
""" Provides the path of a replica for non-deterministic sites. Will be assigned to get path by the __init__ method if necessary.
|
|
339
403
|
|
|
340
404
|
:param scope: list of DIDs
|
|
341
405
|
|
|
@@ -360,7 +424,7 @@ class RSEProtocol(object):
|
|
|
360
424
|
return self.translator.path(scope, name)
|
|
361
425
|
|
|
362
426
|
def _get_path_nondeterministic_server(self, scope, name): # pylint: disable=invalid-name
|
|
363
|
-
""" Provides the path of a replica for non-deterministic sites. Will be assigned to get path by the __init__ method if
|
|
427
|
+
""" Provides the path of a replica for non-deterministic sites. Will be assigned to get path by the __init__ method if necessary. """
|
|
364
428
|
vo = get_rse_vo(self.rse['id']) # pylint: disable=E0601
|
|
365
429
|
scope = InternalScope(scope, vo=vo) # pylint: disable=E0601
|
|
366
430
|
rep = replica.get_replica(scope=scope, name=name, rse_id=self.rse['id']) # pylint: disable=E0601
|
|
@@ -378,7 +442,7 @@ class RSEProtocol(object):
|
|
|
378
442
|
|
|
379
443
|
def parse_pfns(self, pfns):
|
|
380
444
|
"""
|
|
381
|
-
Splits the given PFN into the parts known by the protocol. It is also checked if the provided protocol
|
|
445
|
+
Splits the given PFN into the parts known by the protocol. It is also checked if the provided protocol supports the given PFNs.
|
|
382
446
|
|
|
383
447
|
:param pfns: a list of a fully qualified PFNs
|
|
384
448
|
|
|
@@ -418,7 +482,7 @@ class RSEProtocol(object):
|
|
|
418
482
|
raise exception.RSEFileNameNotSupported('Invalid prefix: provided \'%s\', expected \'%s\'' % ('/'.join(path.split('/')[0:len(prefix.split('/')) - 1]),
|
|
419
483
|
prefix)) # len(...)-1 due to the leading '/
|
|
420
484
|
|
|
421
|
-
#
|
|
485
|
+
# Splitting parsed.path into prefix, path, filename
|
|
422
486
|
path = path.partition(prefix)[2]
|
|
423
487
|
name = path.split('/')[-1]
|
|
424
488
|
path = '/'.join(path.split('/')[:-1])
|
|
@@ -463,7 +527,7 @@ class RSEProtocol(object):
|
|
|
463
527
|
:param transfer_timeout: Transfer timeout (in seconds)
|
|
464
528
|
|
|
465
529
|
:raises DestinationNotAccessible: if the destination storage was not accessible.
|
|
466
|
-
:raises ServiceUnavailable: if some generic error
|
|
530
|
+
:raises ServiceUnavailable: if some generic error occurred in the library.
|
|
467
531
|
:raises SourceNotFound: if the source file was not found on the referred storage.
|
|
468
532
|
"""
|
|
469
533
|
raise NotImplementedError
|
|
@@ -478,7 +542,7 @@ class RSEProtocol(object):
|
|
|
478
542
|
:param transfer_timeout: Transfer timeout (in seconds)
|
|
479
543
|
|
|
480
544
|
:raises DestinationNotAccessible: if the destination storage was not accessible.
|
|
481
|
-
:raises ServiceUnavailable: if some generic error
|
|
545
|
+
:raises ServiceUnavailable: if some generic error occurred in the library.
|
|
482
546
|
:raises SourceNotFound: if the source file was not found on the referred storage.
|
|
483
547
|
"""
|
|
484
548
|
raise NotImplementedError
|
|
@@ -489,7 +553,7 @@ class RSEProtocol(object):
|
|
|
489
553
|
|
|
490
554
|
:param path: path to the to be deleted file
|
|
491
555
|
|
|
492
|
-
:raises ServiceUnavailable: if some generic error
|
|
556
|
+
:raises ServiceUnavailable: if some generic error occurred in the library.
|
|
493
557
|
:raises SourceNotFound: if the source file was not found on the referred storage.
|
|
494
558
|
"""
|
|
495
559
|
raise NotImplementedError
|
|
@@ -501,7 +565,7 @@ class RSEProtocol(object):
|
|
|
501
565
|
:param new_path: path to the new file on the storage
|
|
502
566
|
|
|
503
567
|
:raises DestinationNotAccessible: if the destination storage was not accessible.
|
|
504
|
-
:raises ServiceUnavailable: if some generic error
|
|
568
|
+
:raises ServiceUnavailable: if some generic error occurred in the library.
|
|
505
569
|
:raises SourceNotFound: if the source file was not found on the referred storage.
|
|
506
570
|
"""
|
|
507
571
|
raise NotImplementedError
|
|
@@ -512,7 +576,7 @@ class RSEProtocol(object):
|
|
|
512
576
|
|
|
513
577
|
:returns: a list with dict containing 'totalsize' and 'unusedsize'
|
|
514
578
|
|
|
515
|
-
:raises ServiceUnavailable: if some generic error
|
|
579
|
+
:raises ServiceUnavailable: if some generic error occurred in the library.
|
|
516
580
|
"""
|
|
517
581
|
raise NotImplementedError
|
|
518
582
|
|
|
@@ -522,7 +586,7 @@ class RSEProtocol(object):
|
|
|
522
586
|
|
|
523
587
|
:param path: path to file
|
|
524
588
|
|
|
525
|
-
:raises ServiceUnavailable: if some generic error
|
|
589
|
+
:raises ServiceUnavailable: if some generic error occurred in the library.
|
|
526
590
|
:raises SourceNotFound: if the source file was not found on the referred storage.
|
|
527
591
|
|
|
528
592
|
:returns: a dict with two keys, filesize and adler32 of the file provided in path.
|
rucio/rse/protocols/rclone.py
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
1
|
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
3
2
|
#
|
|
4
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -19,7 +18,7 @@ import os
|
|
|
19
18
|
|
|
20
19
|
from rucio.common import exception
|
|
21
20
|
from rucio.common.config import get_config_dirs
|
|
22
|
-
from rucio.common.utils import
|
|
21
|
+
from rucio.common.utils import PREFERRED_CHECKSUM, execute
|
|
23
22
|
from rucio.rse.protocols import protocol
|
|
24
23
|
|
|
25
24
|
|
|
@@ -169,7 +168,7 @@ class Default(protocol.RSEProtocol):
|
|
|
169
168
|
|
|
170
169
|
:param path: path to file
|
|
171
170
|
|
|
172
|
-
:raises ServiceUnavailable: if some generic error
|
|
171
|
+
:raises ServiceUnavailable: if some generic error occurred in the library.
|
|
173
172
|
|
|
174
173
|
:returns: a dict with two keys, filesize and an element of GLOBALLY_SUPPORTED_CHECKSUMS.
|
|
175
174
|
"""
|
|
@@ -242,7 +241,7 @@ class Default(protocol.RSEProtocol):
|
|
|
242
241
|
if not prefix.endswith('/'):
|
|
243
242
|
prefix = ''.join([prefix, '/'])
|
|
244
243
|
|
|
245
|
-
lfns = [lfns] if
|
|
244
|
+
lfns = [lfns] if isinstance(lfns, dict) else lfns
|
|
246
245
|
for lfn in lfns:
|
|
247
246
|
scope, name = lfn['scope'], lfn['name']
|
|
248
247
|
if 'path' in lfn and lfn['path'] is not None:
|
|
@@ -299,7 +298,7 @@ class Default(protocol.RSEProtocol):
|
|
|
299
298
|
:param transfer_timeout: Transfer timeout (in seconds) - dummy
|
|
300
299
|
|
|
301
300
|
:raises DestinationNotAccessible: if the destination storage was not accessible.
|
|
302
|
-
:raises ServiceUnavailable: if some generic error
|
|
301
|
+
:raises ServiceUnavailable: if some generic error occurred in the library.
|
|
303
302
|
:raises SourceNotFound: if the source file was not found on the referred storage.
|
|
304
303
|
"""
|
|
305
304
|
self.logger(logging.DEBUG, 'rclone.put: filename: {} target: {}'.format(filename, target))
|
|
@@ -325,7 +324,7 @@ class Default(protocol.RSEProtocol):
|
|
|
325
324
|
|
|
326
325
|
:param pfn: Physical file name
|
|
327
326
|
|
|
328
|
-
:raises ServiceUnavailable: if some generic error
|
|
327
|
+
:raises ServiceUnavailable: if some generic error occurred in the library.
|
|
329
328
|
:raises SourceNotFound: if the source file was not found on the referred storage.
|
|
330
329
|
"""
|
|
331
330
|
self.logger(logging.DEBUG, 'rclone.delete: pfn: {}'.format(pfn))
|
|
@@ -347,7 +346,7 @@ class Default(protocol.RSEProtocol):
|
|
|
347
346
|
:param pfn: Current physical file name
|
|
348
347
|
:param new_pfn New physical file name
|
|
349
348
|
:raises DestinationNotAccessible: if the destination storage was not accessible.
|
|
350
|
-
:raises ServiceUnavailable: if some generic error
|
|
349
|
+
:raises ServiceUnavailable: if some generic error occurred in the library.
|
|
351
350
|
:raises SourceNotFound: if the source file was not found on the referred storage.
|
|
352
351
|
"""
|
|
353
352
|
self.logger(logging.DEBUG, 'rclone.rename: pfn: {}'.format(pfn))
|
rucio/rse/protocols/rfio.py
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
1
|
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
3
2
|
#
|
|
4
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -33,7 +32,7 @@ class Default(protocol.RSEProtocol):
|
|
|
33
32
|
"""
|
|
34
33
|
Establishes the actual connection to the referred RSE.
|
|
35
34
|
|
|
36
|
-
:param credentials: needed to establish a connection with the
|
|
35
|
+
:param credentials: needed to establish a connection with the storage.
|
|
37
36
|
|
|
38
37
|
:raises RSEAccessDenied: if no connection could be established.
|
|
39
38
|
"""
|
|
@@ -43,7 +42,7 @@ class Default(protocol.RSEProtocol):
|
|
|
43
42
|
|
|
44
43
|
def path2pfn(self, path):
|
|
45
44
|
"""
|
|
46
|
-
|
|
45
|
+
Returns a fully qualified PFN for the file referred by path.
|
|
47
46
|
|
|
48
47
|
:param path: The path to the file.
|
|
49
48
|
|
|
@@ -81,7 +80,7 @@ class Default(protocol.RSEProtocol):
|
|
|
81
80
|
:param transfer_timeout: Transfer timeout (in seconds) - dummy
|
|
82
81
|
|
|
83
82
|
:raises DestinationNotAccessible: if the destination storage was not accessible.
|
|
84
|
-
:raises ServiceUnavailable: if some generic error
|
|
83
|
+
:raises ServiceUnavailable: if some generic error occurred in the library.
|
|
85
84
|
:raises SourceNotFound: if the source file was not found on the referred storage.
|
|
86
85
|
"""
|
|
87
86
|
if not self.exists(dirname(target)):
|
|
@@ -128,7 +127,7 @@ class Default(protocol.RSEProtocol):
|
|
|
128
127
|
if not ret['path'].startswith(self.rse['prefix']):
|
|
129
128
|
raise exception.RSEFileNameNotSupported('Invalid prefix: provided \'%s\', expected \'%s\'' % ('/'.join(ret['path'].split('/')[0:len(self.rse['prefix'].split('/')) - 1]),
|
|
130
129
|
self.rse['prefix'])) # len(...)-1 due to the leading '/
|
|
131
|
-
#
|
|
130
|
+
# Splitting parsed.path into prefix, path, filename
|
|
132
131
|
ret['prefix'] = self.rse['prefix']
|
|
133
132
|
ret['path'] = ret['path'].partition(self.rse['prefix'])[2]
|
|
134
133
|
ret['name'] = ret['path'].split('/')[-1]
|
rucio/rse/protocols/srm.py
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
1
|
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
3
2
|
#
|
|
4
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -51,7 +50,7 @@ class Default(protocol.RSEProtocol):
|
|
|
51
50
|
if '://' in hostname:
|
|
52
51
|
hostname = hostname.split("://")[1]
|
|
53
52
|
|
|
54
|
-
lfns = [lfns] if
|
|
53
|
+
lfns = [lfns] if isinstance(lfns, dict) else lfns
|
|
55
54
|
if not self.attributes['port']:
|
|
56
55
|
for lfn in lfns:
|
|
57
56
|
scope, name, path = lfn['scope'], lfn['name'], lfn.get('path')
|
|
@@ -120,7 +119,7 @@ class Default(protocol.RSEProtocol):
|
|
|
120
119
|
raise exception.RSEFileNameNotSupported('Invalid prefix: provided \'%s\', expected \'%s\'' % ('/'.join(path.split('/')[0:len(self.attributes['prefix'].split('/')) - 1]),
|
|
121
120
|
self.attributes['prefix'])) # len(...)-1 due to the leading '/
|
|
122
121
|
|
|
123
|
-
#
|
|
122
|
+
# Splitting path into prefix, path, filename
|
|
124
123
|
prefix = self.attributes['prefix']
|
|
125
124
|
path = path.partition(self.attributes['prefix'])[2]
|
|
126
125
|
name = path.split('/')[-1]
|
|
@@ -168,17 +167,17 @@ class Default(protocol.RSEProtocol):
|
|
|
168
167
|
def connect(self):
|
|
169
168
|
"""
|
|
170
169
|
Establishes the actual connection to the referred RSE.
|
|
171
|
-
As a quick and dirty
|
|
170
|
+
As a quick and dirty implementation we just use this method to check if the lcg tools are available.
|
|
172
171
|
If we decide to use gfal, init should be done here.
|
|
173
172
|
|
|
174
173
|
:raises RSEAccessDenied: Cannot connect.
|
|
175
174
|
"""
|
|
176
175
|
|
|
177
|
-
status, lcglscommand = getstatusoutput('which lcg-ls')
|
|
176
|
+
status, lcglscommand = getstatusoutput('which lcg-ls') # noqa: S605, S607
|
|
178
177
|
if status:
|
|
179
178
|
raise exception.RSEAccessDenied('Cannot find lcg tools')
|
|
180
179
|
endpoint_basepath = self.path2pfn(self.attributes['prefix'])
|
|
181
|
-
status, result = getstatusoutput('%s -vv $LCGVO -b --srm-timeout 60 -D srmv2 -l %s' % (lcglscommand, endpoint_basepath))
|
|
180
|
+
status, result = getstatusoutput('%s -vv $LCGVO -b --srm-timeout 60 -D srmv2 -l %s' % (lcglscommand, endpoint_basepath)) # noqa: S605
|
|
182
181
|
if status:
|
|
183
182
|
if result == '':
|
|
184
183
|
raise exception.RSEAccessDenied('Endpoint not reachable. lcg-ls failed with status code %s but no further details.' % (str(status)))
|
|
@@ -194,7 +193,7 @@ class Default(protocol.RSEProtocol):
|
|
|
194
193
|
:param transfer_timeout: Transfer timeout (in seconds)
|
|
195
194
|
|
|
196
195
|
:raises DestinationNotAccessible: if the destination storage was not accessible.
|
|
197
|
-
:raises ServiceUnavailable: if some generic error
|
|
196
|
+
:raises ServiceUnavailable: if some generic error occurred in the library.
|
|
198
197
|
:raises SourceNotFound: if the source file was not found on the referred storage.
|
|
199
198
|
"""
|
|
200
199
|
|
|
@@ -224,7 +223,7 @@ class Default(protocol.RSEProtocol):
|
|
|
224
223
|
:param transfer_timeout: Transfer timeout (in seconds)
|
|
225
224
|
|
|
226
225
|
:raises DestinationNotAccessible: if the destination storage was not accessible.
|
|
227
|
-
:raises ServiceUnavailable: if some generic error
|
|
226
|
+
:raises ServiceUnavailable: if some generic error occurred in the library.
|
|
228
227
|
:raises SourceNotFound: if the source file was not found on the referred storage.
|
|
229
228
|
"""
|
|
230
229
|
|
|
@@ -254,7 +253,7 @@ class Default(protocol.RSEProtocol):
|
|
|
254
253
|
Deletes a file from the connected RSE.
|
|
255
254
|
|
|
256
255
|
:param path: path to the to be deleted file
|
|
257
|
-
:raises ServiceUnavailable: if some generic error
|
|
256
|
+
:raises ServiceUnavailable: if some generic error occurred in the library.
|
|
258
257
|
:raises SourceNotFound: if the source file was not found on the referred storage.
|
|
259
258
|
"""
|
|
260
259
|
|
|
@@ -283,7 +282,7 @@ class Default(protocol.RSEProtocol):
|
|
|
283
282
|
:param path: path to the current file on the storage
|
|
284
283
|
:param new_path: path to the new file on the storage
|
|
285
284
|
:raises DestinationNotAccessible: if the destination storage was not accessible.
|
|
286
|
-
:raises ServiceUnavailable: if some generic error
|
|
285
|
+
:raises ServiceUnavailable: if some generic error occurred in the library.
|
|
287
286
|
:raises SourceNotFound: if the source file was not found on the referred storage.
|
|
288
287
|
"""
|
|
289
288
|
|
rucio/rse/protocols/ssh.py
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
1
|
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
3
2
|
#
|
|
4
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -18,7 +17,7 @@ import os
|
|
|
18
17
|
import re
|
|
19
18
|
|
|
20
19
|
from rucio.common import exception
|
|
21
|
-
from rucio.common.utils import
|
|
20
|
+
from rucio.common.utils import PREFERRED_CHECKSUM, execute
|
|
22
21
|
from rucio.rse.protocols import protocol
|
|
23
22
|
|
|
24
23
|
|
|
@@ -86,7 +85,7 @@ class Default(protocol.RSEProtocol):
|
|
|
86
85
|
|
|
87
86
|
:param path: path to file
|
|
88
87
|
|
|
89
|
-
:raises ServiceUnavailable: if some generic error
|
|
88
|
+
:raises ServiceUnavailable: if some generic error occurred in the library.
|
|
90
89
|
|
|
91
90
|
:returns: a dict with two keys, filesize and an element of GLOBALLY_SUPPORTED_CHECKSUMS.
|
|
92
91
|
"""
|
|
@@ -157,7 +156,7 @@ class Default(protocol.RSEProtocol):
|
|
|
157
156
|
if not prefix.endswith('/'):
|
|
158
157
|
prefix = ''.join([prefix, '/'])
|
|
159
158
|
|
|
160
|
-
lfns = [lfns] if
|
|
159
|
+
lfns = [lfns] if isinstance(lfns, dict) else lfns
|
|
161
160
|
for lfn in lfns:
|
|
162
161
|
scope, name = lfn['scope'], lfn['name']
|
|
163
162
|
if 'path' in lfn and lfn['path'] is not None:
|
|
@@ -219,7 +218,7 @@ class Default(protocol.RSEProtocol):
|
|
|
219
218
|
:param transfer_timeout: Transfer timeout (in seconds) - dummy
|
|
220
219
|
|
|
221
220
|
:raises DestinationNotAccessible: if the destination storage was not accessible.
|
|
222
|
-
:raises ServiceUnavailable: if some generic error
|
|
221
|
+
:raises ServiceUnavailable: if some generic error occurred in the library.
|
|
223
222
|
:raises SourceNotFound: if the source file was not found on the referred storage.
|
|
224
223
|
"""
|
|
225
224
|
self.logger(logging.DEBUG, 'ssh.put: filename: {} target: {}'.format(filename, target))
|
|
@@ -246,7 +245,7 @@ class Default(protocol.RSEProtocol):
|
|
|
246
245
|
|
|
247
246
|
:param pfn: Physical file name
|
|
248
247
|
|
|
249
|
-
:raises ServiceUnavailable: if some generic error
|
|
248
|
+
:raises ServiceUnavailable: if some generic error occurred in the library.
|
|
250
249
|
:raises SourceNotFound: if the source file was not found on the referred storage.
|
|
251
250
|
"""
|
|
252
251
|
self.logger(logging.DEBUG, 'ssh.delete: pfn: {}'.format(pfn))
|
|
@@ -268,7 +267,7 @@ class Default(protocol.RSEProtocol):
|
|
|
268
267
|
:param pfn: Current physical file name
|
|
269
268
|
:param new_pfn New physical file name
|
|
270
269
|
:raises DestinationNotAccessible: if the destination storage was not accessible.
|
|
271
|
-
:raises ServiceUnavailable: if some generic error
|
|
270
|
+
:raises ServiceUnavailable: if some generic error occurred in the library.
|
|
272
271
|
:raises SourceNotFound: if the source file was not found on the referred storage.
|
|
273
272
|
"""
|
|
274
273
|
self.logger(logging.DEBUG, 'ssh.rename: pfn: {}'.format(pfn))
|
|
@@ -299,7 +298,7 @@ class Rsync(Default):
|
|
|
299
298
|
|
|
300
299
|
:param path: path to file
|
|
301
300
|
|
|
302
|
-
:raises ServiceUnavailable: if some generic error
|
|
301
|
+
:raises ServiceUnavailable: if some generic error occurred in the library.
|
|
303
302
|
|
|
304
303
|
:returns: a dict with two keys, filesize and an element of GLOBALLY_SUPPORTED_CHECKSUMS.
|
|
305
304
|
"""
|
|
@@ -391,7 +390,7 @@ class Rsync(Default):
|
|
|
391
390
|
:param transfer_timeout: Transfer timeout (in seconds) - dummy
|
|
392
391
|
|
|
393
392
|
:raises DestinationNotAccessible: if the destination storage was not accessible.
|
|
394
|
-
:raises ServiceUnavailable: if some generic error
|
|
393
|
+
:raises ServiceUnavailable: if some generic error occurred in the library.
|
|
395
394
|
:raises SourceNotFound: if the source file was not found on the referred storage.
|
|
396
395
|
"""
|
|
397
396
|
self.logger(logging.DEBUG, 'rsync.put: filename: {} target: {}'.format(filename, target))
|
rucio/rse/protocols/storm.py
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
1
|
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
3
2
|
#
|
|
4
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -60,7 +59,7 @@ class Default(protocol.RSEProtocol):
|
|
|
60
59
|
|
|
61
60
|
def path2pfn(self, path):
|
|
62
61
|
"""
|
|
63
|
-
|
|
62
|
+
Returns a fully qualified PFN for the file referred by path.
|
|
64
63
|
|
|
65
64
|
:param path: The path to the file.
|
|
66
65
|
|
|
@@ -156,7 +155,7 @@ class Default(protocol.RSEProtocol):
|
|
|
156
155
|
if rcode != 207:
|
|
157
156
|
rcode, etag_meta = davix_etag(pfn, 300)
|
|
158
157
|
|
|
159
|
-
p_output = minidom.parseString(etag_meta)
|
|
158
|
+
p_output = minidom.parseString(etag_meta) # noqa: S318
|
|
160
159
|
# we need to strip off the quotation marks and the <timestamp> from the etag
|
|
161
160
|
# but since we can have multiple underscores, we have to rely on the uniqueness
|
|
162
161
|
# of the full LFN to make the split
|