rucio-clients 32.8.6__py3-none-any.whl → 35.8.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of rucio-clients might be problematic. Click here for more details.

Files changed (92) hide show
  1. rucio/__init__.py +0 -1
  2. rucio/alembicrevision.py +1 -2
  3. rucio/client/__init__.py +0 -1
  4. rucio/client/accountclient.py +45 -25
  5. rucio/client/accountlimitclient.py +37 -9
  6. rucio/client/baseclient.py +199 -154
  7. rucio/client/client.py +2 -3
  8. rucio/client/configclient.py +19 -6
  9. rucio/client/credentialclient.py +9 -4
  10. rucio/client/didclient.py +238 -63
  11. rucio/client/diracclient.py +13 -5
  12. rucio/client/downloadclient.py +162 -51
  13. rucio/client/exportclient.py +4 -4
  14. rucio/client/fileclient.py +3 -4
  15. rucio/client/importclient.py +4 -4
  16. rucio/client/lifetimeclient.py +21 -5
  17. rucio/client/lockclient.py +18 -8
  18. rucio/client/{metaclient.py → metaconventionsclient.py} +18 -15
  19. rucio/client/pingclient.py +0 -1
  20. rucio/client/replicaclient.py +15 -5
  21. rucio/client/requestclient.py +35 -19
  22. rucio/client/rseclient.py +133 -51
  23. rucio/client/ruleclient.py +29 -22
  24. rucio/client/scopeclient.py +8 -6
  25. rucio/client/subscriptionclient.py +47 -35
  26. rucio/client/touchclient.py +8 -4
  27. rucio/client/uploadclient.py +166 -82
  28. rucio/common/__init__.py +0 -1
  29. rucio/common/cache.py +4 -4
  30. rucio/common/config.py +52 -47
  31. rucio/common/constants.py +69 -2
  32. rucio/common/constraints.py +0 -1
  33. rucio/common/didtype.py +24 -22
  34. rucio/common/exception.py +281 -222
  35. rucio/common/extra.py +0 -1
  36. rucio/common/logging.py +54 -38
  37. rucio/common/pcache.py +122 -101
  38. rucio/common/plugins.py +153 -0
  39. rucio/common/policy.py +4 -4
  40. rucio/common/schema/__init__.py +17 -10
  41. rucio/common/schema/atlas.py +7 -5
  42. rucio/common/schema/belleii.py +7 -5
  43. rucio/common/schema/domatpc.py +7 -5
  44. rucio/common/schema/escape.py +7 -5
  45. rucio/common/schema/generic.py +8 -6
  46. rucio/common/schema/generic_multi_vo.py +7 -5
  47. rucio/common/schema/icecube.py +7 -5
  48. rucio/common/stomp_utils.py +0 -1
  49. rucio/common/stopwatch.py +0 -1
  50. rucio/common/test_rucio_server.py +2 -2
  51. rucio/common/types.py +262 -17
  52. rucio/common/utils.py +743 -451
  53. rucio/rse/__init__.py +3 -4
  54. rucio/rse/protocols/__init__.py +0 -1
  55. rucio/rse/protocols/bittorrent.py +184 -0
  56. rucio/rse/protocols/cache.py +1 -2
  57. rucio/rse/protocols/dummy.py +1 -2
  58. rucio/rse/protocols/gfal.py +12 -10
  59. rucio/rse/protocols/globus.py +7 -7
  60. rucio/rse/protocols/gsiftp.py +2 -3
  61. rucio/rse/protocols/http_cache.py +1 -2
  62. rucio/rse/protocols/mock.py +1 -2
  63. rucio/rse/protocols/ngarc.py +1 -2
  64. rucio/rse/protocols/posix.py +12 -13
  65. rucio/rse/protocols/protocol.py +116 -52
  66. rucio/rse/protocols/rclone.py +6 -7
  67. rucio/rse/protocols/rfio.py +4 -5
  68. rucio/rse/protocols/srm.py +9 -10
  69. rucio/rse/protocols/ssh.py +8 -9
  70. rucio/rse/protocols/storm.py +2 -3
  71. rucio/rse/protocols/webdav.py +17 -14
  72. rucio/rse/protocols/xrootd.py +23 -17
  73. rucio/rse/rsemanager.py +19 -7
  74. rucio/vcsversion.py +4 -4
  75. rucio/version.py +5 -13
  76. rucio_clients-35.8.0.data/data/requirements.client.txt +15 -0
  77. {rucio_clients-32.8.6.data → rucio_clients-35.8.0.data}/data/rucio_client/merge_rucio_configs.py +2 -5
  78. {rucio_clients-32.8.6.data → rucio_clients-35.8.0.data}/scripts/rucio +87 -85
  79. {rucio_clients-32.8.6.data → rucio_clients-35.8.0.data}/scripts/rucio-admin +45 -32
  80. {rucio_clients-32.8.6.dist-info → rucio_clients-35.8.0.dist-info}/METADATA +13 -13
  81. rucio_clients-35.8.0.dist-info/RECORD +88 -0
  82. {rucio_clients-32.8.6.dist-info → rucio_clients-35.8.0.dist-info}/WHEEL +1 -1
  83. {rucio_clients-32.8.6.dist-info → rucio_clients-35.8.0.dist-info}/licenses/AUTHORS.rst +3 -0
  84. rucio/common/schema/cms.py +0 -478
  85. rucio/common/schema/lsst.py +0 -423
  86. rucio_clients-32.8.6.data/data/requirements.txt +0 -55
  87. rucio_clients-32.8.6.dist-info/RECORD +0 -88
  88. {rucio_clients-32.8.6.data → rucio_clients-35.8.0.data}/data/etc/rse-accounts.cfg.template +0 -0
  89. {rucio_clients-32.8.6.data → rucio_clients-35.8.0.data}/data/etc/rucio.cfg.atlas.client.template +0 -0
  90. {rucio_clients-32.8.6.data → rucio_clients-35.8.0.data}/data/etc/rucio.cfg.template +0 -0
  91. {rucio_clients-32.8.6.dist-info → rucio_clients-35.8.0.dist-info}/licenses/LICENSE +0 -0
  92. {rucio_clients-32.8.6.dist-info → rucio_clients-35.8.0.dist-info}/top_level.txt +0 -0
@@ -1,4 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
1
  # Copyright European Organization for Nuclear Research (CERN) since 2012
3
2
  #
4
3
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -17,14 +16,16 @@
17
16
  This module defines the base class for implementing a transfer protocol,
18
17
  along with some of the default methods for LFN2PFN translations.
19
18
  """
20
-
21
19
  import hashlib
22
20
  import logging
21
+ from collections.abc import Callable, Mapping
23
22
  from configparser import NoOptionError, NoSectionError
23
+ from typing import Any, Optional, TypeVar
24
24
  from urllib.parse import urlparse
25
25
 
26
26
  from rucio.common import config, exception
27
- from rucio.common.utils import register_policy_package_algorithms
27
+ from rucio.common.constants import RseAttr
28
+ from rucio.common.plugins import PolicyPackageAlgorithms
28
29
  from rucio.rse import rsemanager
29
30
 
30
31
  if getattr(rsemanager, 'CLIENT_MODE', None):
@@ -36,13 +37,102 @@ if getattr(rsemanager, 'SERVER_MODE', None):
36
37
  from rucio.core.rse import get_rse_vo
37
38
 
38
39
 
39
- class RSEDeterministicTranslation(object):
40
+ class RSEDeterministicScopeTranslation(PolicyPackageAlgorithms):
41
+ """
42
+ Translates a pfn dictionary into a scope and name
43
+ """
44
+
45
+ _algorithm_type = "pfn2lfn"
46
+
47
+ def __init__(self, vo: str = 'def'):
48
+ super().__init__()
49
+
50
+ self.register(RSEDeterministicScopeTranslation._default, "def")
51
+ self.register(RSEDeterministicScopeTranslation._atlas, "atlas")
52
+
53
+ logger = logging.getLogger(__name__)
54
+
55
+ try:
56
+ # Use the function defined in the policy package if it's configured so
57
+ algorithm_name = config.config_get('policy', self._algorithm_type)
58
+ except (NoOptionError, NoSectionError, RuntimeError):
59
+ # Don't use a function from the policy package. Use one defined in this class according to vo
60
+ logger.debug("PFN2LFN function will not be fetched from the policy package")
61
+ if super()._supports(self._algorithm_type, vo):
62
+ algorithm_name = vo
63
+ else:
64
+ algorithm_name = "def"
65
+
66
+ self.parser = self.get_parser(algorithm_name)
67
+
68
+ @classmethod
69
+ def get_parser(cls, algorithm_name: str) -> Callable[..., Any]:
70
+ return super()._get_one_algorithm(cls._algorithm_type, algorithm_name)
71
+
72
+ @classmethod
73
+ def register(
74
+ cls,
75
+ pfn2lfn_callable: Callable,
76
+ name: Optional[str] = None
77
+ ) -> None:
78
+ """
79
+ Provided a callable function, register it as one of the valid PFN2LFN algorithms.
80
+
81
+
82
+ :param pfn2lfn_callable: Callable function to use.
83
+ :param name: Algorithm name used for registration.
84
+ """
85
+ if name is None:
86
+ name = pfn2lfn_callable.__name__
87
+ algorithm_dict = {name: pfn2lfn_callable}
88
+ super()._register(cls._algorithm_type, algorithm_dict)
89
+
90
+ @staticmethod
91
+ def _default(parsed_pfn: Mapping[str, str]) -> tuple[str, str]:
92
+ """ Translate pfn to name/scope pair
93
+
94
+ :param parsed_pfn: dictionary representing pfn containing:
95
+ - path: str,
96
+ - name: str
97
+ :return: tuple containing name, scope
98
+ """
99
+ path = parsed_pfn['path']
100
+ scope = path.lstrip('/').split('/')[0]
101
+ name = parsed_pfn['name']
102
+ return name, scope
103
+
104
+ @staticmethod
105
+ def _atlas(parsed_pfn: Mapping[str, str]) -> tuple[str, str]:
106
+ """ Translate pfn to name/scope pair
107
+
108
+ :param parsed_pfn: dictionary representing pfn containing:
109
+ - path: str,
110
+ - name: str
111
+ :return: tuple containing name, scope
112
+ """
113
+ path = parsed_pfn['path']
114
+ if path.startswith('/user') or path.startswith('/group'):
115
+ scope = '%s.%s' % (path.split('/')[1], path.split('/')[2])
116
+ name = parsed_pfn['name']
117
+ else:
118
+ name, scope = RSEDeterministicScopeTranslation._default(parsed_pfn)
119
+
120
+ return name, scope
121
+
122
+
123
+ RSEDeterministicScopeTranslation()
124
+
125
+
126
+ RSEDeterministicTranslationT = TypeVar('RSEDeterministicTranslationT', bound='RSEDeterministicTranslation')
127
+
128
+
129
+ class RSEDeterministicTranslation(PolicyPackageAlgorithms):
40
130
  """
41
131
  Execute the logic for translating a LFN to a path.
42
132
  """
43
133
 
44
- _LFN2PFN_ALGORITHMS = {}
45
134
  _DEFAULT_LFN2PFN = "hash"
135
+ _algorithm_type = "lfn2pfn"
46
136
 
47
137
  def __init__(self, rse=None, rse_attributes=None, protocol_attributes=None):
48
138
  """
@@ -53,10 +143,10 @@ class RSEDeterministicTranslation(object):
53
143
  :param rse_attributes: A dictionary of RSE-specific attributes for use in the translation.
54
144
  :param protocol_attributes: A dictionary of RSE/protocol-specific attributes.
55
145
  """
146
+ super().__init__()
56
147
  self.rse = rse
57
148
  self.rse_attributes = rse_attributes if rse_attributes else {}
58
149
  self.protocol_attributes = protocol_attributes if protocol_attributes else {}
59
- self.loaded_policy_modules = False
60
150
 
61
151
  @classmethod
62
152
  def supports(cls, name):
@@ -64,12 +154,12 @@ class RSEDeterministicTranslation(object):
64
154
  Check to see if a specific algorithm is supported.
65
155
 
66
156
  :param name: Name of the deterministic algorithm.
67
- :returns: True if `name` is an algorithm supported by the translator class, False otherwise.
157
+ :returns: True if `name` is an algorithm supported by the translator class, False otherwise
68
158
  """
69
- return name in cls._LFN2PFN_ALGORITHMS
159
+ return super()._supports(cls._algorithm_type, name)
70
160
 
71
- @staticmethod
72
- def register(lfn2pfn_callable, name=None):
161
+ @classmethod
162
+ def register(cls, lfn2pfn_callable, name=None):
73
163
  """
74
164
  Provided a callable function, register it as one of the valid LFN2PFN algorithms.
75
165
 
@@ -87,7 +177,8 @@ class RSEDeterministicTranslation(object):
87
177
  """
88
178
  if name is None:
89
179
  name = lfn2pfn_callable.__name__
90
- RSEDeterministicTranslation._LFN2PFN_ALGORITHMS[name] = lfn2pfn_callable
180
+ algorithm_dict = {name: lfn2pfn_callable}
181
+ super()._register(cls._algorithm_type, algorithm_dict)
91
182
 
92
183
  @staticmethod
93
184
  def __hash(scope, name, rse, rse_attrs, protocol_attrs):
@@ -194,27 +285,6 @@ class RSEDeterministicTranslation(object):
194
285
 
195
286
  return '%s/%s/%s/%s' % (scope[0:7], scope[4:len(scope)], name.split('-')[0] + "-" + name.split('-')[1], name)
196
287
 
197
- @staticmethod
198
- def __lsst(scope, name, rse, rse_attrs, protocol_attrs):
199
- """
200
- LFN2PFN algorithm for Rubin-LSST in the ESCAPE project
201
-
202
- Replace convention delimiter '__' by '/'
203
- The Escape instance does use the 'generic' Rucio schema.
204
-
205
- :param scope: Scope of the LFN (ignored)
206
- :param name: File name of the LFN.
207
- :param rse: RSE for PFN (ignored)
208
- :param rse_attrs: RSE attributes for PFN (ignored)
209
- :param protocol_attrs: RSE protocol attributes for PFN (ignored)
210
- :returns: Path for use in the PFN generation.
211
- """
212
- del scope
213
- del rse
214
- del rse_attrs
215
- del protocol_attrs
216
- return name.replace('__', '/')
217
-
218
288
  @classmethod
219
289
  def _module_init_(cls):
220
290
  """
@@ -225,7 +295,6 @@ class RSEDeterministicTranslation(object):
225
295
  cls.register(cls.__ligo, "ligo")
226
296
  cls.register(cls.__belleii, "belleii")
227
297
  cls.register(cls.__xenon, "xenon")
228
- cls.register(cls.__lsst, "lsst")
229
298
  policy_module = None
230
299
  try:
231
300
  policy_module = config.config_get('policy', 'lfn2pfn_module')
@@ -247,22 +316,17 @@ class RSEDeterministicTranslation(object):
247
316
 
248
317
  :returns: RSE specific URI of the physical file
249
318
  """
250
- # on first call, register any lfn2pfn algorithms from the policy package(s) (server only)
251
- if getattr(rsemanager, 'SERVER_MODE', None) and not self.loaded_policy_modules:
252
- register_policy_package_algorithms('lfn2pfn', RSEDeterministicTranslation._LFN2PFN_ALGORITHMS)
253
- self.loaded_policy_modules = True
254
-
255
- algorithm = self.rse_attributes.get('lfn2pfn_algorithm', 'default')
319
+ algorithm = self.rse_attributes.get(RseAttr.LFN2PFN_ALGORITHM, 'default')
256
320
  if algorithm == 'default':
257
321
  algorithm = RSEDeterministicTranslation._DEFAULT_LFN2PFN
258
- algorithm_callable = RSEDeterministicTranslation._LFN2PFN_ALGORITHMS[algorithm]
322
+ algorithm_callable = super()._get_one_algorithm(RSEDeterministicTranslation._algorithm_type, algorithm)
259
323
  return algorithm_callable(scope, name, self.rse, self.rse_attributes, self.protocol_attributes)
260
324
 
261
325
 
262
326
  RSEDeterministicTranslation._module_init_() # pylint: disable=protected-access
263
327
 
264
328
 
265
- class RSEProtocol(object):
329
+ class RSEProtocol:
266
330
  """ This class is virtual and acts as a base to inherit new protocols from. It further provides some common functionality which applies for the amjority of the protocols."""
267
331
 
268
332
  def __init__(self, protocol_attr, rse_settings, logger=logging.log):
@@ -294,7 +358,7 @@ class RSEProtocol(object):
294
358
 
295
359
  def lfns2pfns(self, lfns):
296
360
  """
297
- Retruns a fully qualified PFN for the file referred by path.
361
+ Returns a fully qualified PFN for the file referred by path.
298
362
 
299
363
  :param path: The path to the file.
300
364
 
@@ -335,7 +399,7 @@ class RSEProtocol(object):
335
399
  return pfns
336
400
 
337
401
  def __lfns2pfns_client(self, lfns):
338
- """ Provides the path of a replica for non-deterministic sites. Will be assigned to get path by the __init__ method if neccessary.
402
+ """ Provides the path of a replica for non-deterministic sites. Will be assigned to get path by the __init__ method if necessary.
339
403
 
340
404
  :param scope: list of DIDs
341
405
 
@@ -360,7 +424,7 @@ class RSEProtocol(object):
360
424
  return self.translator.path(scope, name)
361
425
 
362
426
  def _get_path_nondeterministic_server(self, scope, name): # pylint: disable=invalid-name
363
- """ Provides the path of a replica for non-deterministic sites. Will be assigned to get path by the __init__ method if neccessary. """
427
+ """ Provides the path of a replica for non-deterministic sites. Will be assigned to get path by the __init__ method if necessary. """
364
428
  vo = get_rse_vo(self.rse['id']) # pylint: disable=E0601
365
429
  scope = InternalScope(scope, vo=vo) # pylint: disable=E0601
366
430
  rep = replica.get_replica(scope=scope, name=name, rse_id=self.rse['id']) # pylint: disable=E0601
@@ -378,7 +442,7 @@ class RSEProtocol(object):
378
442
 
379
443
  def parse_pfns(self, pfns):
380
444
  """
381
- Splits the given PFN into the parts known by the protocol. It is also checked if the provided protocol supportes the given PFNs.
445
+ Splits the given PFN into the parts known by the protocol. It is also checked if the provided protocol supports the given PFNs.
382
446
 
383
447
  :param pfns: a list of a fully qualified PFNs
384
448
 
@@ -418,7 +482,7 @@ class RSEProtocol(object):
418
482
  raise exception.RSEFileNameNotSupported('Invalid prefix: provided \'%s\', expected \'%s\'' % ('/'.join(path.split('/')[0:len(prefix.split('/')) - 1]),
419
483
  prefix)) # len(...)-1 due to the leading '/
420
484
 
421
- # Spliting parsed.path into prefix, path, filename
485
+ # Splitting parsed.path into prefix, path, filename
422
486
  path = path.partition(prefix)[2]
423
487
  name = path.split('/')[-1]
424
488
  path = '/'.join(path.split('/')[:-1])
@@ -463,7 +527,7 @@ class RSEProtocol(object):
463
527
  :param transfer_timeout: Transfer timeout (in seconds)
464
528
 
465
529
  :raises DestinationNotAccessible: if the destination storage was not accessible.
466
- :raises ServiceUnavailable: if some generic error occured in the library.
530
+ :raises ServiceUnavailable: if some generic error occurred in the library.
467
531
  :raises SourceNotFound: if the source file was not found on the referred storage.
468
532
  """
469
533
  raise NotImplementedError
@@ -478,7 +542,7 @@ class RSEProtocol(object):
478
542
  :param transfer_timeout: Transfer timeout (in seconds)
479
543
 
480
544
  :raises DestinationNotAccessible: if the destination storage was not accessible.
481
- :raises ServiceUnavailable: if some generic error occured in the library.
545
+ :raises ServiceUnavailable: if some generic error occurred in the library.
482
546
  :raises SourceNotFound: if the source file was not found on the referred storage.
483
547
  """
484
548
  raise NotImplementedError
@@ -489,7 +553,7 @@ class RSEProtocol(object):
489
553
 
490
554
  :param path: path to the to be deleted file
491
555
 
492
- :raises ServiceUnavailable: if some generic error occured in the library.
556
+ :raises ServiceUnavailable: if some generic error occurred in the library.
493
557
  :raises SourceNotFound: if the source file was not found on the referred storage.
494
558
  """
495
559
  raise NotImplementedError
@@ -501,7 +565,7 @@ class RSEProtocol(object):
501
565
  :param new_path: path to the new file on the storage
502
566
 
503
567
  :raises DestinationNotAccessible: if the destination storage was not accessible.
504
- :raises ServiceUnavailable: if some generic error occured in the library.
568
+ :raises ServiceUnavailable: if some generic error occurred in the library.
505
569
  :raises SourceNotFound: if the source file was not found on the referred storage.
506
570
  """
507
571
  raise NotImplementedError
@@ -512,7 +576,7 @@ class RSEProtocol(object):
512
576
 
513
577
  :returns: a list with dict containing 'totalsize' and 'unusedsize'
514
578
 
515
- :raises ServiceUnavailable: if some generic error occured in the library.
579
+ :raises ServiceUnavailable: if some generic error occurred in the library.
516
580
  """
517
581
  raise NotImplementedError
518
582
 
@@ -522,7 +586,7 @@ class RSEProtocol(object):
522
586
 
523
587
  :param path: path to file
524
588
 
525
- :raises ServiceUnavailable: if some generic error occured in the library.
589
+ :raises ServiceUnavailable: if some generic error occurred in the library.
526
590
  :raises SourceNotFound: if the source file was not found on the referred storage.
527
591
 
528
592
  :returns: a dict with two keys, filesize and adler32 of the file provided in path.
@@ -1,4 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
1
  # Copyright European Organization for Nuclear Research (CERN) since 2012
3
2
  #
4
3
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -19,7 +18,7 @@ import os
19
18
 
20
19
  from rucio.common import exception
21
20
  from rucio.common.config import get_config_dirs
22
- from rucio.common.utils import execute, PREFERRED_CHECKSUM
21
+ from rucio.common.utils import PREFERRED_CHECKSUM, execute
23
22
  from rucio.rse.protocols import protocol
24
23
 
25
24
 
@@ -169,7 +168,7 @@ class Default(protocol.RSEProtocol):
169
168
 
170
169
  :param path: path to file
171
170
 
172
- :raises ServiceUnavailable: if some generic error occured in the library.
171
+ :raises ServiceUnavailable: if some generic error occurred in the library.
173
172
 
174
173
  :returns: a dict with two keys, filesize and an element of GLOBALLY_SUPPORTED_CHECKSUMS.
175
174
  """
@@ -242,7 +241,7 @@ class Default(protocol.RSEProtocol):
242
241
  if not prefix.endswith('/'):
243
242
  prefix = ''.join([prefix, '/'])
244
243
 
245
- lfns = [lfns] if type(lfns) == dict else lfns
244
+ lfns = [lfns] if isinstance(lfns, dict) else lfns
246
245
  for lfn in lfns:
247
246
  scope, name = lfn['scope'], lfn['name']
248
247
  if 'path' in lfn and lfn['path'] is not None:
@@ -299,7 +298,7 @@ class Default(protocol.RSEProtocol):
299
298
  :param transfer_timeout: Transfer timeout (in seconds) - dummy
300
299
 
301
300
  :raises DestinationNotAccessible: if the destination storage was not accessible.
302
- :raises ServiceUnavailable: if some generic error occured in the library.
301
+ :raises ServiceUnavailable: if some generic error occurred in the library.
303
302
  :raises SourceNotFound: if the source file was not found on the referred storage.
304
303
  """
305
304
  self.logger(logging.DEBUG, 'rclone.put: filename: {} target: {}'.format(filename, target))
@@ -325,7 +324,7 @@ class Default(protocol.RSEProtocol):
325
324
 
326
325
  :param pfn: Physical file name
327
326
 
328
- :raises ServiceUnavailable: if some generic error occured in the library.
327
+ :raises ServiceUnavailable: if some generic error occurred in the library.
329
328
  :raises SourceNotFound: if the source file was not found on the referred storage.
330
329
  """
331
330
  self.logger(logging.DEBUG, 'rclone.delete: pfn: {}'.format(pfn))
@@ -347,7 +346,7 @@ class Default(protocol.RSEProtocol):
347
346
  :param pfn: Current physical file name
348
347
  :param new_pfn New physical file name
349
348
  :raises DestinationNotAccessible: if the destination storage was not accessible.
350
- :raises ServiceUnavailable: if some generic error occured in the library.
349
+ :raises ServiceUnavailable: if some generic error occurred in the library.
351
350
  :raises SourceNotFound: if the source file was not found on the referred storage.
352
351
  """
353
352
  self.logger(logging.DEBUG, 'rclone.rename: pfn: {}'.format(pfn))
@@ -1,4 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
1
  # Copyright European Organization for Nuclear Research (CERN) since 2012
3
2
  #
4
3
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -33,7 +32,7 @@ class Default(protocol.RSEProtocol):
33
32
  """
34
33
  Establishes the actual connection to the referred RSE.
35
34
 
36
- :param credentials: needed to establish a connection with the stroage.
35
+ :param credentials: needed to establish a connection with the storage.
37
36
 
38
37
  :raises RSEAccessDenied: if no connection could be established.
39
38
  """
@@ -43,7 +42,7 @@ class Default(protocol.RSEProtocol):
43
42
 
44
43
  def path2pfn(self, path):
45
44
  """
46
- Retruns a fully qualified PFN for the file referred by path.
45
+ Returns a fully qualified PFN for the file referred by path.
47
46
 
48
47
  :param path: The path to the file.
49
48
 
@@ -81,7 +80,7 @@ class Default(protocol.RSEProtocol):
81
80
  :param transfer_timeout: Transfer timeout (in seconds) - dummy
82
81
 
83
82
  :raises DestinationNotAccessible: if the destination storage was not accessible.
84
- :raises ServiceUnavailable: if some generic error occured in the library.
83
+ :raises ServiceUnavailable: if some generic error occurred in the library.
85
84
  :raises SourceNotFound: if the source file was not found on the referred storage.
86
85
  """
87
86
  if not self.exists(dirname(target)):
@@ -128,7 +127,7 @@ class Default(protocol.RSEProtocol):
128
127
  if not ret['path'].startswith(self.rse['prefix']):
129
128
  raise exception.RSEFileNameNotSupported('Invalid prefix: provided \'%s\', expected \'%s\'' % ('/'.join(ret['path'].split('/')[0:len(self.rse['prefix'].split('/')) - 1]),
130
129
  self.rse['prefix'])) # len(...)-1 due to the leading '/
131
- # Spliting parsed.path into prefix, path, filename
130
+ # Splitting parsed.path into prefix, path, filename
132
131
  ret['prefix'] = self.rse['prefix']
133
132
  ret['path'] = ret['path'].partition(self.rse['prefix'])[2]
134
133
  ret['name'] = ret['path'].split('/')[-1]
@@ -1,4 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
1
  # Copyright European Organization for Nuclear Research (CERN) since 2012
3
2
  #
4
3
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -51,7 +50,7 @@ class Default(protocol.RSEProtocol):
51
50
  if '://' in hostname:
52
51
  hostname = hostname.split("://")[1]
53
52
 
54
- lfns = [lfns] if type(lfns) == dict else lfns
53
+ lfns = [lfns] if isinstance(lfns, dict) else lfns
55
54
  if not self.attributes['port']:
56
55
  for lfn in lfns:
57
56
  scope, name, path = lfn['scope'], lfn['name'], lfn.get('path')
@@ -120,7 +119,7 @@ class Default(protocol.RSEProtocol):
120
119
  raise exception.RSEFileNameNotSupported('Invalid prefix: provided \'%s\', expected \'%s\'' % ('/'.join(path.split('/')[0:len(self.attributes['prefix'].split('/')) - 1]),
121
120
  self.attributes['prefix'])) # len(...)-1 due to the leading '/
122
121
 
123
- # Spliting path into prefix, path, filename
122
+ # Splitting path into prefix, path, filename
124
123
  prefix = self.attributes['prefix']
125
124
  path = path.partition(self.attributes['prefix'])[2]
126
125
  name = path.split('/')[-1]
@@ -168,17 +167,17 @@ class Default(protocol.RSEProtocol):
168
167
  def connect(self):
169
168
  """
170
169
  Establishes the actual connection to the referred RSE.
171
- As a quick and dirty impelementation we just use this method to check if the lcg tools are available.
170
+ As a quick and dirty implementation we just use this method to check if the lcg tools are available.
172
171
  If we decide to use gfal, init should be done here.
173
172
 
174
173
  :raises RSEAccessDenied: Cannot connect.
175
174
  """
176
175
 
177
- status, lcglscommand = getstatusoutput('which lcg-ls')
176
+ status, lcglscommand = getstatusoutput('which lcg-ls') # noqa: S605, S607
178
177
  if status:
179
178
  raise exception.RSEAccessDenied('Cannot find lcg tools')
180
179
  endpoint_basepath = self.path2pfn(self.attributes['prefix'])
181
- status, result = getstatusoutput('%s -vv $LCGVO -b --srm-timeout 60 -D srmv2 -l %s' % (lcglscommand, endpoint_basepath))
180
+ status, result = getstatusoutput('%s -vv $LCGVO -b --srm-timeout 60 -D srmv2 -l %s' % (lcglscommand, endpoint_basepath)) # noqa: S605
182
181
  if status:
183
182
  if result == '':
184
183
  raise exception.RSEAccessDenied('Endpoint not reachable. lcg-ls failed with status code %s but no further details.' % (str(status)))
@@ -194,7 +193,7 @@ class Default(protocol.RSEProtocol):
194
193
  :param transfer_timeout: Transfer timeout (in seconds)
195
194
 
196
195
  :raises DestinationNotAccessible: if the destination storage was not accessible.
197
- :raises ServiceUnavailable: if some generic error occured in the library.
196
+ :raises ServiceUnavailable: if some generic error occurred in the library.
198
197
  :raises SourceNotFound: if the source file was not found on the referred storage.
199
198
  """
200
199
 
@@ -224,7 +223,7 @@ class Default(protocol.RSEProtocol):
224
223
  :param transfer_timeout: Transfer timeout (in seconds)
225
224
 
226
225
  :raises DestinationNotAccessible: if the destination storage was not accessible.
227
- :raises ServiceUnavailable: if some generic error occured in the library.
226
+ :raises ServiceUnavailable: if some generic error occurred in the library.
228
227
  :raises SourceNotFound: if the source file was not found on the referred storage.
229
228
  """
230
229
 
@@ -254,7 +253,7 @@ class Default(protocol.RSEProtocol):
254
253
  Deletes a file from the connected RSE.
255
254
 
256
255
  :param path: path to the to be deleted file
257
- :raises ServiceUnavailable: if some generic error occured in the library.
256
+ :raises ServiceUnavailable: if some generic error occurred in the library.
258
257
  :raises SourceNotFound: if the source file was not found on the referred storage.
259
258
  """
260
259
 
@@ -283,7 +282,7 @@ class Default(protocol.RSEProtocol):
283
282
  :param path: path to the current file on the storage
284
283
  :param new_path: path to the new file on the storage
285
284
  :raises DestinationNotAccessible: if the destination storage was not accessible.
286
- :raises ServiceUnavailable: if some generic error occured in the library.
285
+ :raises ServiceUnavailable: if some generic error occurred in the library.
287
286
  :raises SourceNotFound: if the source file was not found on the referred storage.
288
287
  """
289
288
 
@@ -1,4 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
1
  # Copyright European Organization for Nuclear Research (CERN) since 2012
3
2
  #
4
3
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -18,7 +17,7 @@ import os
18
17
  import re
19
18
 
20
19
  from rucio.common import exception
21
- from rucio.common.utils import execute, PREFERRED_CHECKSUM
20
+ from rucio.common.utils import PREFERRED_CHECKSUM, execute
22
21
  from rucio.rse.protocols import protocol
23
22
 
24
23
 
@@ -86,7 +85,7 @@ class Default(protocol.RSEProtocol):
86
85
 
87
86
  :param path: path to file
88
87
 
89
- :raises ServiceUnavailable: if some generic error occured in the library.
88
+ :raises ServiceUnavailable: if some generic error occurred in the library.
90
89
 
91
90
  :returns: a dict with two keys, filesize and an element of GLOBALLY_SUPPORTED_CHECKSUMS.
92
91
  """
@@ -157,7 +156,7 @@ class Default(protocol.RSEProtocol):
157
156
  if not prefix.endswith('/'):
158
157
  prefix = ''.join([prefix, '/'])
159
158
 
160
- lfns = [lfns] if type(lfns) == dict else lfns
159
+ lfns = [lfns] if isinstance(lfns, dict) else lfns
161
160
  for lfn in lfns:
162
161
  scope, name = lfn['scope'], lfn['name']
163
162
  if 'path' in lfn and lfn['path'] is not None:
@@ -219,7 +218,7 @@ class Default(protocol.RSEProtocol):
219
218
  :param transfer_timeout: Transfer timeout (in seconds) - dummy
220
219
 
221
220
  :raises DestinationNotAccessible: if the destination storage was not accessible.
222
- :raises ServiceUnavailable: if some generic error occured in the library.
221
+ :raises ServiceUnavailable: if some generic error occurred in the library.
223
222
  :raises SourceNotFound: if the source file was not found on the referred storage.
224
223
  """
225
224
  self.logger(logging.DEBUG, 'ssh.put: filename: {} target: {}'.format(filename, target))
@@ -246,7 +245,7 @@ class Default(protocol.RSEProtocol):
246
245
 
247
246
  :param pfn: Physical file name
248
247
 
249
- :raises ServiceUnavailable: if some generic error occured in the library.
248
+ :raises ServiceUnavailable: if some generic error occurred in the library.
250
249
  :raises SourceNotFound: if the source file was not found on the referred storage.
251
250
  """
252
251
  self.logger(logging.DEBUG, 'ssh.delete: pfn: {}'.format(pfn))
@@ -268,7 +267,7 @@ class Default(protocol.RSEProtocol):
268
267
  :param pfn: Current physical file name
269
268
  :param new_pfn New physical file name
270
269
  :raises DestinationNotAccessible: if the destination storage was not accessible.
271
- :raises ServiceUnavailable: if some generic error occured in the library.
270
+ :raises ServiceUnavailable: if some generic error occurred in the library.
272
271
  :raises SourceNotFound: if the source file was not found on the referred storage.
273
272
  """
274
273
  self.logger(logging.DEBUG, 'ssh.rename: pfn: {}'.format(pfn))
@@ -299,7 +298,7 @@ class Rsync(Default):
299
298
 
300
299
  :param path: path to file
301
300
 
302
- :raises ServiceUnavailable: if some generic error occured in the library.
301
+ :raises ServiceUnavailable: if some generic error occurred in the library.
303
302
 
304
303
  :returns: a dict with two keys, filesize and an element of GLOBALLY_SUPPORTED_CHECKSUMS.
305
304
  """
@@ -391,7 +390,7 @@ class Rsync(Default):
391
390
  :param transfer_timeout: Transfer timeout (in seconds) - dummy
392
391
 
393
392
  :raises DestinationNotAccessible: if the destination storage was not accessible.
394
- :raises ServiceUnavailable: if some generic error occured in the library.
393
+ :raises ServiceUnavailable: if some generic error occurred in the library.
395
394
  :raises SourceNotFound: if the source file was not found on the referred storage.
396
395
  """
397
396
  self.logger(logging.DEBUG, 'rsync.put: filename: {} target: {}'.format(filename, target))
@@ -1,4 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
1
  # Copyright European Organization for Nuclear Research (CERN) since 2012
3
2
  #
4
3
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -60,7 +59,7 @@ class Default(protocol.RSEProtocol):
60
59
 
61
60
  def path2pfn(self, path):
62
61
  """
63
- Retruns a fully qualified PFN for the file referred by path.
62
+ Returns a fully qualified PFN for the file referred by path.
64
63
 
65
64
  :param path: The path to the file.
66
65
 
@@ -156,7 +155,7 @@ class Default(protocol.RSEProtocol):
156
155
  if rcode != 207:
157
156
  rcode, etag_meta = davix_etag(pfn, 300)
158
157
 
159
- p_output = minidom.parseString(etag_meta)
158
+ p_output = minidom.parseString(etag_meta) # noqa: S318
160
159
  # we need to strip off the quotation marks and the <timestamp> from the etag
161
160
  # but since we can have multiple underscores, we have to rely on the uniqueness
162
161
  # of the full LFN to make the split