rucio-clients 35.7.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of rucio-clients might be problematic. Click here for more details.

Files changed (88) hide show
  1. rucio/__init__.py +17 -0
  2. rucio/alembicrevision.py +15 -0
  3. rucio/client/__init__.py +15 -0
  4. rucio/client/accountclient.py +433 -0
  5. rucio/client/accountlimitclient.py +183 -0
  6. rucio/client/baseclient.py +974 -0
  7. rucio/client/client.py +76 -0
  8. rucio/client/configclient.py +126 -0
  9. rucio/client/credentialclient.py +59 -0
  10. rucio/client/didclient.py +866 -0
  11. rucio/client/diracclient.py +56 -0
  12. rucio/client/downloadclient.py +1785 -0
  13. rucio/client/exportclient.py +44 -0
  14. rucio/client/fileclient.py +50 -0
  15. rucio/client/importclient.py +42 -0
  16. rucio/client/lifetimeclient.py +90 -0
  17. rucio/client/lockclient.py +109 -0
  18. rucio/client/metaconventionsclient.py +140 -0
  19. rucio/client/pingclient.py +44 -0
  20. rucio/client/replicaclient.py +454 -0
  21. rucio/client/requestclient.py +125 -0
  22. rucio/client/rseclient.py +746 -0
  23. rucio/client/ruleclient.py +294 -0
  24. rucio/client/scopeclient.py +90 -0
  25. rucio/client/subscriptionclient.py +173 -0
  26. rucio/client/touchclient.py +82 -0
  27. rucio/client/uploadclient.py +955 -0
  28. rucio/common/__init__.py +13 -0
  29. rucio/common/cache.py +74 -0
  30. rucio/common/config.py +801 -0
  31. rucio/common/constants.py +159 -0
  32. rucio/common/constraints.py +17 -0
  33. rucio/common/didtype.py +189 -0
  34. rucio/common/exception.py +1151 -0
  35. rucio/common/extra.py +36 -0
  36. rucio/common/logging.py +420 -0
  37. rucio/common/pcache.py +1408 -0
  38. rucio/common/plugins.py +153 -0
  39. rucio/common/policy.py +84 -0
  40. rucio/common/schema/__init__.py +150 -0
  41. rucio/common/schema/atlas.py +413 -0
  42. rucio/common/schema/belleii.py +408 -0
  43. rucio/common/schema/domatpc.py +401 -0
  44. rucio/common/schema/escape.py +426 -0
  45. rucio/common/schema/generic.py +433 -0
  46. rucio/common/schema/generic_multi_vo.py +412 -0
  47. rucio/common/schema/icecube.py +406 -0
  48. rucio/common/stomp_utils.py +159 -0
  49. rucio/common/stopwatch.py +55 -0
  50. rucio/common/test_rucio_server.py +148 -0
  51. rucio/common/types.py +403 -0
  52. rucio/common/utils.py +2238 -0
  53. rucio/rse/__init__.py +96 -0
  54. rucio/rse/protocols/__init__.py +13 -0
  55. rucio/rse/protocols/bittorrent.py +184 -0
  56. rucio/rse/protocols/cache.py +122 -0
  57. rucio/rse/protocols/dummy.py +111 -0
  58. rucio/rse/protocols/gfal.py +703 -0
  59. rucio/rse/protocols/globus.py +243 -0
  60. rucio/rse/protocols/gsiftp.py +92 -0
  61. rucio/rse/protocols/http_cache.py +82 -0
  62. rucio/rse/protocols/mock.py +123 -0
  63. rucio/rse/protocols/ngarc.py +209 -0
  64. rucio/rse/protocols/posix.py +250 -0
  65. rucio/rse/protocols/protocol.py +594 -0
  66. rucio/rse/protocols/rclone.py +364 -0
  67. rucio/rse/protocols/rfio.py +136 -0
  68. rucio/rse/protocols/srm.py +338 -0
  69. rucio/rse/protocols/ssh.py +413 -0
  70. rucio/rse/protocols/storm.py +206 -0
  71. rucio/rse/protocols/webdav.py +550 -0
  72. rucio/rse/protocols/xrootd.py +301 -0
  73. rucio/rse/rsemanager.py +764 -0
  74. rucio/vcsversion.py +11 -0
  75. rucio/version.py +38 -0
  76. rucio_clients-35.7.0.data/data/etc/rse-accounts.cfg.template +25 -0
  77. rucio_clients-35.7.0.data/data/etc/rucio.cfg.atlas.client.template +42 -0
  78. rucio_clients-35.7.0.data/data/etc/rucio.cfg.template +257 -0
  79. rucio_clients-35.7.0.data/data/requirements.client.txt +15 -0
  80. rucio_clients-35.7.0.data/data/rucio_client/merge_rucio_configs.py +144 -0
  81. rucio_clients-35.7.0.data/scripts/rucio +2542 -0
  82. rucio_clients-35.7.0.data/scripts/rucio-admin +2447 -0
  83. rucio_clients-35.7.0.dist-info/METADATA +50 -0
  84. rucio_clients-35.7.0.dist-info/RECORD +88 -0
  85. rucio_clients-35.7.0.dist-info/WHEEL +5 -0
  86. rucio_clients-35.7.0.dist-info/licenses/AUTHORS.rst +97 -0
  87. rucio_clients-35.7.0.dist-info/licenses/LICENSE +201 -0
  88. rucio_clients-35.7.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,413 @@
1
+ # Copyright European Organization for Nuclear Research (CERN) since 2012
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ import logging
16
+ import os
17
+ import re
18
+
19
+ from rucio.common import exception
20
+ from rucio.common.utils import PREFERRED_CHECKSUM, execute
21
+ from rucio.rse.protocols import protocol
22
+
23
+
24
+ class Default(protocol.RSEProtocol):
25
+ """ Implementing access to RSEs using the SSH protocol."""
26
+
27
+ def __init__(self, protocol_attr, rse_settings, logger=logging.log):
28
+ """ Initializes the object with information about the referred RSE.
29
+
30
+ :param props: Properties derived from the RSE Repository
31
+ """
32
+ super(Default, self).__init__(protocol_attr, rse_settings, logger=logger)
33
+
34
+ self.scheme = self.attributes['scheme']
35
+ self.hostname = self.attributes['hostname']
36
+ self.port = str(self.attributes['port'])
37
+ self.path = None
38
+ if self.attributes['extended_attributes'] is not None and\
39
+ 'user' in list(self.attributes['extended_attributes'].keys()):
40
+ self.sshuser = self.attributes['extended_attributes']['user'] + '@'
41
+ else:
42
+ self.sshuser = ''
43
+ self.logger = logger
44
+
45
+ def path2pfn(self, path):
46
+ """
47
+ Returns a fully qualified PFN for the file referred by path.
48
+
49
+ :param path: The path to the file.
50
+
51
+ :returns: Fully qualified PFN.
52
+
53
+ """
54
+ self.logger(logging.DEBUG, 'ssh.path2pfn: path: {}'.format(path))
55
+ if not path.startswith(str(self.scheme) + '://'):
56
+ return '%s://%s%s:%s/%s' % (self.scheme, self.sshuser, self.hostname, self.port, path)
57
+ else:
58
+ return path
59
+
60
+ def exists(self, pfn):
61
+ """ Checks if the requested file is known by the referred RSE.
62
+
63
+ :param pfn: Physical file name
64
+
65
+ :returns: True if the file exists, False if it doesn't
66
+
67
+ :raise ServiceUnavailable
68
+ """
69
+ self.logger(logging.DEBUG, 'ssh.exists: pfn: {}'.format(pfn))
70
+ try:
71
+ path = self.pfn2path(pfn)
72
+ cmd = 'ssh -p %s %s%s find %s' % (self.port, self.sshuser, self.hostname, path)
73
+ self.logger(logging.DEBUG, 'ssh.exists: cmd: {}'.format(cmd))
74
+ status, out, err = execute(cmd)
75
+ if status:
76
+ return False
77
+ except Exception as e:
78
+ raise exception.ServiceUnavailable(e)
79
+
80
+ return True
81
+
82
+ def stat(self, path):
83
+ """
84
+ Returns the stats of a file.
85
+
86
+ :param path: path to file
87
+
88
+ :raises ServiceUnavailable: if some generic error occurred in the library.
89
+
90
+ :returns: a dict with two keys, filesize and an element of GLOBALLY_SUPPORTED_CHECKSUMS.
91
+ """
92
+ self.logger(logging.DEBUG, 'ssh.stat: path: {}'.format(path))
93
+ ret = {}
94
+ chsum = None
95
+ path = self.pfn2path(path)
96
+
97
+ try:
98
+ # ssh stat for getting filesize
99
+ cmd = 'ssh -p {0} {1}{2} stat --printf="%s" {3}'.format(self.port, self.sshuser, self.hostname, path)
100
+ self.logger(logging.DEBUG, 'ssh.stat: filesize cmd: {}'.format(cmd))
101
+ status_stat, out, err = execute(cmd)
102
+ if status_stat == 0:
103
+ ret['filesize'] = out
104
+
105
+ # ssh query checksum for getting md5 checksum
106
+ cmd = 'ssh -p %s %s%s md5sum %s' % (self.port, self.sshuser, self.hostname, path)
107
+ self.logger(logging.DEBUG, 'ssh.stat: checksum cmd: {}'.format(cmd))
108
+ status_query, out, err = execute(cmd)
109
+
110
+ if status_query == 0:
111
+ chsum = 'md5'
112
+ val = out.strip(' ').split()
113
+ ret[chsum] = val[0]
114
+
115
+ except Exception as e:
116
+ raise exception.ServiceUnavailable(e)
117
+
118
+ if 'filesize' not in ret:
119
+ raise exception.ServiceUnavailable('Filesize could not be retrieved.')
120
+ if PREFERRED_CHECKSUM != chsum or not chsum:
121
+ msg = '{} does not match with {}'.format(chsum, PREFERRED_CHECKSUM)
122
+ raise exception.RSEChecksumUnavailable(msg)
123
+
124
+ return ret
125
+
126
+ def pfn2path(self, pfn):
127
+ """
128
+ Returns the path of a file given the pfn, i.e. scheme, user and hostname are subtracted from the pfn.
129
+
130
+ :param path: pfn of a file
131
+
132
+ :returns: path.
133
+ """
134
+ path = pfn
135
+ if pfn.startswith(str(self.scheme) + '://'):
136
+ self.logger(logging.DEBUG, 'ssh.pfn2path: pfn: {}'.format(pfn))
137
+ prefix = self.attributes['prefix']
138
+ path = pfn.partition(self.attributes['prefix'])[2]
139
+ path = prefix + path
140
+ return path
141
+
142
+ def lfns2pfns(self, lfns):
143
+ """
144
+ Returns a fully qualified PFN for the file referred by path.
145
+
146
+ :param path: The path to the file.
147
+
148
+ :returns: Fully qualified PFN.
149
+ """
150
+ self.logger(logging.DEBUG, 'ssh.lfns2pfns: lfns: {}'.format(lfns))
151
+ pfns = {}
152
+ prefix = self.attributes['prefix']
153
+
154
+ if not prefix.startswith('/'):
155
+ prefix = ''.join(['/', prefix])
156
+ if not prefix.endswith('/'):
157
+ prefix = ''.join([prefix, '/'])
158
+
159
+ lfns = [lfns] if isinstance(lfns, dict) else lfns
160
+ for lfn in lfns:
161
+ scope, name = lfn['scope'], lfn['name']
162
+ if 'path' in lfn and lfn['path'] is not None:
163
+ pfns['%s:%s' % (scope, name)] = ''.join([self.attributes['scheme'], '://', self.sshuser, self.hostname, ':', self.port, prefix, lfn['path']])
164
+ else:
165
+ pfns['%s:%s' % (scope, name)] = ''.join([self.attributes['scheme'], '://', self.sshuser, self.hostname, ':', self.port, prefix, self._get_path(scope=scope, name=name)])
166
+ return pfns
167
+
168
+ def connect(self):
169
+ """ Establishes the actual connection to the referred RSE.
170
+
171
+ :raises RSEAccessDenied
172
+ """
173
+ self.logger(logging.DEBUG, 'ssh.connect: port: {}, hostname {}, ssh-user {}'.format(self.port, self.hostname, self.sshuser))
174
+ try:
175
+ cmd = 'ssh -p %s %s%s echo ok 2>&1' % (self.port, self.sshuser, self.hostname)
176
+ status, out, err = execute(cmd)
177
+ checker = re.search(r'ok', out)
178
+ if not checker:
179
+ raise exception.RSEAccessDenied(err)
180
+ except Exception as e:
181
+ raise exception.RSEAccessDenied(e)
182
+
183
+ def close(self):
184
+ """ Closes the connection to RSE."""
185
+ pass
186
+
187
+ def get(self, pfn, dest, transfer_timeout=None):
188
+ """ Provides access to files stored inside connected the RSE.
189
+
190
+ :param pfn: Physical file name of requested file
191
+ :param dest: Name and path of the files when stored at the client
192
+ :param transfer_timeout: Transfer timeout (in seconds) - dummy
193
+
194
+ :raises DestinationNotAccessible, ServiceUnavailable, SourceNotFound
195
+ """
196
+ self.logger(logging.DEBUG, 'ssh.get: pfn: {}'.format(pfn))
197
+ try:
198
+ path = self.pfn2path(pfn)
199
+ destdir = os.path.dirname(dest)
200
+ cmd = 'mkdir -p %s' % (destdir)
201
+ self.logger(logging.DEBUG, 'ssh.get: cmd: {}'.format(cmd))
202
+ status, out, err = execute(cmd)
203
+ cmd = 'scp %s%s:%s %s' % (self.sshuser, self.hostname, path, dest)
204
+ self.logger(logging.DEBUG, 'ssh.get: cmd: {}'.format(cmd))
205
+ status, out, err = execute(cmd)
206
+ if status:
207
+ raise exception.RucioException(err)
208
+ except Exception as e:
209
+ raise exception.ServiceUnavailable(e)
210
+
211
+ def put(self, filename, target, source_dir, transfer_timeout=None):
212
+ """
213
+ Allows to store files inside the referred RSE.
214
+
215
+ :param source: path to the source file on the client file system
216
+ :param target: path to the destination file on the storage
217
+ :param source_dir: Path where the to be transferred files are stored in the local file system
218
+ :param transfer_timeout: Transfer timeout (in seconds) - dummy
219
+
220
+ :raises DestinationNotAccessible: if the destination storage was not accessible.
221
+ :raises ServiceUnavailable: if some generic error occurred in the library.
222
+ :raises SourceNotFound: if the source file was not found on the referred storage.
223
+ """
224
+ self.logger(logging.DEBUG, 'ssh.put: filename: {} target: {}'.format(filename, target))
225
+ source_dir = source_dir or '.'
226
+ source_url = '%s/%s' % (source_dir, filename)
227
+ self.logger(logging.DEBUG, 'ssh.put: source url: {}'.format(source_url))
228
+
229
+ path = self.pfn2path(target)
230
+ pathdir = os.path.dirname(path)
231
+ if not os.path.exists(source_url):
232
+ raise exception.SourceNotFound()
233
+ try:
234
+ cmd = 'ssh %s%s "mkdir -p %s" && scp %s %s%s:%s' % (self.sshuser, self.hostname, pathdir, source_url, self.sshuser, self.hostname, path)
235
+ self.logger(logging.DEBUG, 'ssh.put: cmd: {}'.format(cmd))
236
+ status, out, err = execute(cmd)
237
+ if status:
238
+ raise exception.RucioException(err)
239
+ except Exception as e:
240
+ raise exception.ServiceUnavailable(e)
241
+
242
+ def delete(self, pfn):
243
+ """
244
+ Deletes a file from the connected RSE.
245
+
246
+ :param pfn: Physical file name
247
+
248
+ :raises ServiceUnavailable: if some generic error occurred in the library.
249
+ :raises SourceNotFound: if the source file was not found on the referred storage.
250
+ """
251
+ self.logger(logging.DEBUG, 'ssh.delete: pfn: {}'.format(pfn))
252
+ if not self.exists(pfn):
253
+ raise exception.SourceNotFound()
254
+ try:
255
+ path = self.pfn2path(pfn)
256
+ cmd = 'ssh -p %s %s%s rm %s' % (self.port, self.sshuser, self.hostname, path)
257
+ self.logger(logging.DEBUG, 'ssh.delete: cmd: {}'.format(cmd))
258
+ status, out, err = execute(cmd)
259
+ if status != 0:
260
+ raise exception.RucioException(err)
261
+ except Exception as e:
262
+ raise exception.ServiceUnavailable(e)
263
+
264
+ def rename(self, pfn, new_pfn):
265
+ """ Allows to rename a file stored inside the connected RSE.
266
+
267
+ :param pfn: Current physical file name
268
+ :param new_pfn New physical file name
269
+ :raises DestinationNotAccessible: if the destination storage was not accessible.
270
+ :raises ServiceUnavailable: if some generic error occurred in the library.
271
+ :raises SourceNotFound: if the source file was not found on the referred storage.
272
+ """
273
+ self.logger(logging.DEBUG, 'ssh.rename: pfn: {}'.format(pfn))
274
+ if not self.exists(pfn):
275
+ raise exception.SourceNotFound()
276
+ try:
277
+ path = self.pfn2path(pfn)
278
+ new_path = self.pfn2path(new_pfn)
279
+ new_dir = new_path[:new_path.rindex('/') + 1]
280
+ cmd = 'ssh -p %s %s%s "mkdir -p %s"' % (self.port, self.sshuser, self.hostname, new_dir)
281
+ self.logger(logging.DEBUG, 'ssh.rename: mkdir cmd: {}'.format(cmd))
282
+ status, out, err = execute(cmd)
283
+ cmd = 'ssh -p %s %s%s mv %s %s' % (self.port, self.sshuser, self.hostname, path, new_path)
284
+ self.logger(logging.DEBUG, 'ssh.rename: rename cmd: {}'.format(cmd))
285
+ status, out, err = execute(cmd)
286
+ if status != 0:
287
+ raise exception.RucioException(err)
288
+ except Exception as e:
289
+ raise exception.ServiceUnavailable(e)
290
+
291
+
292
+ class Rsync(Default):
293
+ """ Implementing access to RSEs using the ssh.Rsync implementation."""
294
+
295
+ def stat(self, path):
296
+ """
297
+ Returns the stats of a file.
298
+
299
+ :param path: path to file
300
+
301
+ :raises ServiceUnavailable: if some generic error occurred in the library.
302
+
303
+ :returns: a dict with two keys, filesize and an element of GLOBALLY_SUPPORTED_CHECKSUMS.
304
+ """
305
+ self.logger(logging.DEBUG, 'rsync.stat: path: {}'.format(path))
306
+ ret = {}
307
+ chsum = None
308
+ path = self.pfn2path(path)
309
+
310
+ try:
311
+ # rsync stat for getting filesize
312
+ cmd = "rsync -an --size-only -e 'ssh -p {0}' --remove-source-files {1}{2}:{3}".format(self.port, self.sshuser, self.hostname, path)
313
+ self.logger(logging.DEBUG, 'rsync.stat: filesize cmd: {}'.format(cmd))
314
+ status_stat, out, err = execute(cmd)
315
+ if status_stat == 0:
316
+ sizestr = out.split(" ")[-4]
317
+ ret['filesize'] = sizestr.replace(',', '')
318
+
319
+ # rsync query checksum for getting md5 checksum
320
+ cmd = 'ssh -p %s %s%s md5sum %s' % (self.port, self.sshuser, self.hostname, path)
321
+ self.logger(logging.DEBUG, 'rsync.stat: checksum cmd: {}'.format(cmd))
322
+ status_query, out, err = execute(cmd)
323
+
324
+ if status_query == 0:
325
+ chsum = 'md5'
326
+ val = out.strip(' ').split()
327
+ ret[chsum] = val[0]
328
+
329
+ except Exception as e:
330
+ raise exception.ServiceUnavailable(e)
331
+
332
+ if 'filesize' not in ret:
333
+ raise exception.ServiceUnavailable('Filesize could not be retrieved.')
334
+ if PREFERRED_CHECKSUM != chsum or not chsum:
335
+ msg = '{} does not match with {}'.format(chsum, PREFERRED_CHECKSUM)
336
+ raise exception.RSEChecksumUnavailable(msg)
337
+
338
+ return ret
339
+
340
+ def connect(self):
341
+ """ Establishes the actual connection to the referred RSE.
342
+
343
+ :raises RSEAccessDenied
344
+ """
345
+ self.logger(logging.DEBUG, 'rsync.connect: port: {}, hostname {}, ssh-user {}'.format(self.port, self.hostname, self.sshuser))
346
+ try:
347
+ cmd = 'ssh -p %s %s%s echo ok 2>&1' % (self.port, self.sshuser, self.hostname)
348
+ status, out, err = execute(cmd)
349
+ checker = re.search(r'ok', out)
350
+ if not checker:
351
+ raise exception.RSEAccessDenied(err)
352
+ cmd = 'ssh -p %s %s%s type rsync' % (self.port, self.sshuser, self.hostname)
353
+ status, out, err = execute(cmd)
354
+ checker = re.search(r'rsync is', out)
355
+ if not checker:
356
+ raise exception.RSEAccessDenied(err)
357
+ self.path = out.split(" ")[2][:-1]
358
+
359
+ except Exception as e:
360
+ raise exception.RSEAccessDenied(e)
361
+
362
+ def get(self, pfn, dest, transfer_timeout=None):
363
+ """ Provides access to files stored inside connected the RSE.
364
+
365
+ :param pfn: Physical file name of requested file
366
+ :param dest: Name and path of the files when stored at the client
367
+ :param transfer_timeout: Transfer timeout (in seconds) - dummy
368
+
369
+ :raises DestinationNotAccessible, ServiceUnavailable, SourceNotFound
370
+ """
371
+ self.logger(logging.DEBUG, 'rsync.get: pfn: {}'.format(pfn))
372
+ try:
373
+ path = self.pfn2path(pfn)
374
+ destdir = os.path.dirname(dest)
375
+ cmd = 'mkdir -p %s && rsync -az -e "ssh -p %s" --append-verify %s%s:%s %s' % (destdir, self.port, self.sshuser, self.hostname, path, dest)
376
+ self.logger(logging.DEBUG, 'rsync.get: cmd: {}'.format(cmd))
377
+ status, out, err = execute(cmd)
378
+ if status:
379
+ raise exception.RucioException(err)
380
+ except Exception as e:
381
+ raise exception.ServiceUnavailable(e)
382
+
383
+ def put(self, filename, target, source_dir, transfer_timeout=None):
384
+ """
385
+ Allows to store files inside the referred RSE.
386
+
387
+ :param source: path to the source file on the client file system
388
+ :param target: path to the destination file on the storage
389
+ :param source_dir: Path where the to be transferred files are stored in the local file system
390
+ :param transfer_timeout: Transfer timeout (in seconds) - dummy
391
+
392
+ :raises DestinationNotAccessible: if the destination storage was not accessible.
393
+ :raises ServiceUnavailable: if some generic error occurred in the library.
394
+ :raises SourceNotFound: if the source file was not found on the referred storage.
395
+ """
396
+ self.logger(logging.DEBUG, 'rsync.put: filename: {} target: {}'.format(filename, target))
397
+ source_dir = source_dir or '.'
398
+ source_url = '%s/%s' % (source_dir, filename)
399
+ self.logger(logging.DEBUG, 'rsync.put: source url: {}'.format(source_url))
400
+
401
+ path = self.pfn2path(target)
402
+ pathdir = os.path.dirname(path)
403
+ if not os.path.exists(source_url):
404
+ raise exception.SourceNotFound()
405
+
406
+ try:
407
+ cmd = 'ssh -p %s %s%s "mkdir -p %s" && rsync -az -e "ssh -p %s" --append-verify %s %s%s:%s' % (self.port, self.sshuser, self.hostname, pathdir, self.port, source_url, self.sshuser, self.hostname, path)
408
+ self.logger(logging.DEBUG, 'rsync.put: cmd: {}'.format(cmd))
409
+ status, out, err = execute(cmd)
410
+ if status:
411
+ raise exception.RucioException(err)
412
+ except Exception as e:
413
+ raise exception.ServiceUnavailable(e)
@@ -0,0 +1,206 @@
1
+ # Copyright European Organization for Nuclear Research (CERN) since 2012
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ import logging
16
+ import os
17
+ from xml.dom import minidom
18
+
19
+ import requests
20
+
21
+ from rucio.common import exception
22
+ from rucio.common.utils import run_cmd_process
23
+ from rucio.rse.protocols import protocol
24
+
25
+
26
+ class Default(protocol.RSEProtocol):
27
+ """ Implementing access to RSEs using the local filesystem."""
28
+
29
+ def __init__(self, protocol_attr, rse_settings, logger=None):
30
+ """ Initializes the object with information about the referred RSE.
31
+
32
+ :param props: Properties derived from the RSE Repository
33
+ """
34
+ super(Default, self).__init__(protocol_attr, rse_settings, logger=logger)
35
+ self.attributes.pop('determinism_type', None)
36
+ self.files = []
37
+
38
+ def lfns2pfns(self, lfns):
39
+ """ Create fake storm:// path. Will be resolved at the get() stage later. """
40
+ pfns = {}
41
+
42
+ hostname = self.attributes['hostname']
43
+ if '://' in hostname:
44
+ hostname = hostname.split("://")[1]
45
+
46
+ prefix = self.attributes['prefix']
47
+ if not prefix.startswith('/'):
48
+ prefix = ''.join(['/', prefix])
49
+ if not prefix.endswith('/'):
50
+ prefix = ''.join([prefix, '/'])
51
+
52
+ lfns = [lfns] if isinstance(lfns, dict) else lfns
53
+ for lfn in lfns:
54
+ path = lfn['path'] if 'path' in lfn and lfn['path'] else self._get_path(scope=lfn['scope'],
55
+ name=lfn['name'])
56
+ pfns['%s:%s' % (lfn['scope'], lfn['name'])] = ''.join(['storm://', hostname, ':', str(self.attributes['port']), prefix, path])
57
+
58
+ return pfns
59
+
60
+ def path2pfn(self, path):
61
+ """
62
+ Returns a fully qualified PFN for the file referred by path.
63
+
64
+ :param path: The path to the file.
65
+
66
+ :returns: Fully qualified PFN.
67
+
68
+ """
69
+ return ''.join([self.rse['scheme'], '://%s' % self.rse['hostname'], path])
70
+
71
+ def exists(self, pfn):
72
+ """ Checks if the requested file is known by the referred RSE.
73
+
74
+ :param pfn: Physical file name
75
+
76
+ :returns: True if the file exists, False if it doesn't
77
+
78
+ :raise ServiceUnavailable
79
+ """
80
+ raise NotImplementedError
81
+
82
+ def connect(self):
83
+ """ Establishes the actual connection to the referred RSE.
84
+
85
+ :param credentials: Provide all necessary information to establish a connection
86
+ to the referred storage system. Some is loaded from the repository inside the
87
+ RSE class and some must be provided specific for the SFTP protocol like
88
+ username, password, private_key, private_key_pass, port.
89
+ For details about possible additional parameters and details about their usage
90
+ see the pysftp.Connection() documentation.
91
+ NOTE: the host parametrer is overwritten with the value provided by the repository
92
+
93
+ :raise RSEAccessDenied
94
+ """
95
+ pass
96
+
97
+ def close(self):
98
+ """ Closes the connection to RSE."""
99
+ pass
100
+
101
+ def get(self, pfn, dest, transfer_timeout=None):
102
+ """ Provides access to files stored inside connected the RSE.
103
+
104
+ :param pfn: Physical file name of requested file
105
+ :param dest: Name and path of the files when stored at the client
106
+ :param transfer_timeout Transfer timeout (in seconds)
107
+
108
+ :raises DestinationNotAccessible, ServiceUnavailable, SourceNotFound
109
+ """
110
+
111
+ # retrieve the TURL from the webdav etag, REQUESTS
112
+ def requests_etag(pfn, timeout):
113
+ x509 = os.environ.get('X509_USER_PROXY')
114
+ pfn = 'https' + pfn[:5]
115
+ session = requests.Session()
116
+ output = session.request('PROPFIND', pfn, verify=False, timeout=timeout, cert=(x509, x509))
117
+ session.close()
118
+ return output.status_code, output.text
119
+
120
+ # retrieve the TURL from the webdav etag, DAVIX
121
+ def davix_etag(pfn, timeout):
122
+ pfn = 'davs' + pfn[5:]
123
+ cmd = 'davix-http --capath /cvmfs/atlas.cern.ch/repo/ATLASLocalRootBase/etc/grid-security-emi/certificates --cert $X509_USER_PROXY -X PROPFIND %s' % pfn
124
+ try:
125
+ rcode, output = run_cmd_process(cmd, timeout=timeout)
126
+ if rcode != 0:
127
+ if output:
128
+ raise exception.ServiceUnavailable("{}/n{}".format(str(output), cmd))
129
+ else:
130
+ raise exception.ServiceUnavailable('Error message from subprocess davix-http call is missing./n{}'.format(cmd))
131
+ except Exception as e:
132
+ raise exception.ServiceUnavailable('Could not retrieve STORM WebDAV ETag: {}/n{}'.format(str(e), cmd))
133
+ return rcode, output
134
+
135
+ # using prefix from AGIS primarily (ATLAS spec.)
136
+ target = None
137
+ try:
138
+ target = self.pfn2path(pfn)
139
+ name = pfn.split('/')[-1]
140
+ if name not in target:
141
+ target = None
142
+ except:
143
+ target = None
144
+ pass
145
+
146
+ # if AGIS setting failed
147
+ etag_meta = None
148
+ if not target:
149
+ # requests preferable
150
+ try:
151
+ rcode, etag_meta = requests_etag(pfn, 300)
152
+ except:
153
+ pass
154
+ # fallback to davix
155
+ if rcode != 207:
156
+ rcode, etag_meta = davix_etag(pfn, 300)
157
+
158
+ p_output = minidom.parseString(etag_meta) # noqa: S318
159
+ # we need to strip off the quotation marks and the <timestamp> from the etag
160
+ # but since we can have multiple underscores, we have to rely on the uniqueness
161
+ # of the full LFN to make the split
162
+ target = p_output.getElementsByTagName('d:getetag')[0].childNodes[0].nodeValue.replace('"', '')
163
+ target_ending = '_' + target.split('_')[-1]
164
+ target = target.split(target_ending)[0]
165
+
166
+ # make the symlink
167
+ try:
168
+ os.symlink(target, dest)
169
+ self.logger(logging.INFO, 'StoRM protocol: {}->{}'.format(target, dest))
170
+ except Exception as e:
171
+ exception.ServiceUnavailable('Could not create symlink: %s for target %s' % (str(e), str(target)))
172
+
173
+ def pfn2path(self, pfn):
174
+ tmp = list(self.parse_pfns(pfn).values())[0]
175
+ return '/'.join([tmp['prefix'], tmp['path'], tmp['name']])
176
+
177
+ def put(self, source, target, source_dir=None, transfer_timeout=None):
178
+ """ Allows to store files inside the referred RSE.
179
+
180
+ :param source: Physical file name
181
+ :param target: Name of the file on the storage system e.g. with prefixed scope
182
+ :param source_dir Path where the to be transferred files are stored in the local file system
183
+ :param transfer_timeout Transfer timeout (in seconds)
184
+
185
+ :raises DestinationNotAccessible, ServiceUnavailable, SourceNotFound
186
+ """
187
+ raise NotImplementedError
188
+
189
+ def delete(self, pfn):
190
+ """ Deletes a file from the connected RSE.
191
+
192
+ :param pfn: Physical file name
193
+
194
+ :raises ServiceUnavailable, SourceNotFound
195
+ """
196
+ raise NotImplementedError
197
+
198
+ def rename(self, pfn, new_pfn):
199
+ """ Allows to rename a file stored inside the connected RSE.
200
+
201
+ :param pfn: Current physical file name
202
+ :param new_pfn New physical file name
203
+
204
+ :raises DestinationNotAccessible, ServiceUnavailable, SourceNotFound
205
+ """
206
+ raise NotImplementedError