rucio-clients 32.8.6__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of rucio-clients might be problematic. Click here for more details.

Files changed (88) hide show
  1. rucio/__init__.py +18 -0
  2. rucio/alembicrevision.py +16 -0
  3. rucio/client/__init__.py +16 -0
  4. rucio/client/accountclient.py +413 -0
  5. rucio/client/accountlimitclient.py +155 -0
  6. rucio/client/baseclient.py +929 -0
  7. rucio/client/client.py +77 -0
  8. rucio/client/configclient.py +113 -0
  9. rucio/client/credentialclient.py +54 -0
  10. rucio/client/didclient.py +691 -0
  11. rucio/client/diracclient.py +48 -0
  12. rucio/client/downloadclient.py +1674 -0
  13. rucio/client/exportclient.py +44 -0
  14. rucio/client/fileclient.py +51 -0
  15. rucio/client/importclient.py +42 -0
  16. rucio/client/lifetimeclient.py +74 -0
  17. rucio/client/lockclient.py +99 -0
  18. rucio/client/metaclient.py +137 -0
  19. rucio/client/pingclient.py +45 -0
  20. rucio/client/replicaclient.py +444 -0
  21. rucio/client/requestclient.py +109 -0
  22. rucio/client/rseclient.py +664 -0
  23. rucio/client/ruleclient.py +287 -0
  24. rucio/client/scopeclient.py +88 -0
  25. rucio/client/subscriptionclient.py +161 -0
  26. rucio/client/touchclient.py +78 -0
  27. rucio/client/uploadclient.py +871 -0
  28. rucio/common/__init__.py +14 -0
  29. rucio/common/cache.py +74 -0
  30. rucio/common/config.py +796 -0
  31. rucio/common/constants.py +92 -0
  32. rucio/common/constraints.py +18 -0
  33. rucio/common/didtype.py +187 -0
  34. rucio/common/exception.py +1092 -0
  35. rucio/common/extra.py +37 -0
  36. rucio/common/logging.py +404 -0
  37. rucio/common/pcache.py +1387 -0
  38. rucio/common/policy.py +84 -0
  39. rucio/common/schema/__init__.py +143 -0
  40. rucio/common/schema/atlas.py +411 -0
  41. rucio/common/schema/belleii.py +406 -0
  42. rucio/common/schema/cms.py +478 -0
  43. rucio/common/schema/domatpc.py +399 -0
  44. rucio/common/schema/escape.py +424 -0
  45. rucio/common/schema/generic.py +431 -0
  46. rucio/common/schema/generic_multi_vo.py +410 -0
  47. rucio/common/schema/icecube.py +404 -0
  48. rucio/common/schema/lsst.py +423 -0
  49. rucio/common/stomp_utils.py +160 -0
  50. rucio/common/stopwatch.py +56 -0
  51. rucio/common/test_rucio_server.py +148 -0
  52. rucio/common/types.py +158 -0
  53. rucio/common/utils.py +1946 -0
  54. rucio/rse/__init__.py +97 -0
  55. rucio/rse/protocols/__init__.py +14 -0
  56. rucio/rse/protocols/cache.py +123 -0
  57. rucio/rse/protocols/dummy.py +112 -0
  58. rucio/rse/protocols/gfal.py +701 -0
  59. rucio/rse/protocols/globus.py +243 -0
  60. rucio/rse/protocols/gsiftp.py +93 -0
  61. rucio/rse/protocols/http_cache.py +83 -0
  62. rucio/rse/protocols/mock.py +124 -0
  63. rucio/rse/protocols/ngarc.py +210 -0
  64. rucio/rse/protocols/posix.py +251 -0
  65. rucio/rse/protocols/protocol.py +530 -0
  66. rucio/rse/protocols/rclone.py +365 -0
  67. rucio/rse/protocols/rfio.py +137 -0
  68. rucio/rse/protocols/srm.py +339 -0
  69. rucio/rse/protocols/ssh.py +414 -0
  70. rucio/rse/protocols/storm.py +207 -0
  71. rucio/rse/protocols/webdav.py +547 -0
  72. rucio/rse/protocols/xrootd.py +295 -0
  73. rucio/rse/rsemanager.py +752 -0
  74. rucio/vcsversion.py +11 -0
  75. rucio/version.py +46 -0
  76. rucio_clients-32.8.6.data/data/etc/rse-accounts.cfg.template +25 -0
  77. rucio_clients-32.8.6.data/data/etc/rucio.cfg.atlas.client.template +42 -0
  78. rucio_clients-32.8.6.data/data/etc/rucio.cfg.template +257 -0
  79. rucio_clients-32.8.6.data/data/requirements.txt +55 -0
  80. rucio_clients-32.8.6.data/data/rucio_client/merge_rucio_configs.py +147 -0
  81. rucio_clients-32.8.6.data/scripts/rucio +2540 -0
  82. rucio_clients-32.8.6.data/scripts/rucio-admin +2434 -0
  83. rucio_clients-32.8.6.dist-info/METADATA +50 -0
  84. rucio_clients-32.8.6.dist-info/RECORD +88 -0
  85. rucio_clients-32.8.6.dist-info/WHEEL +5 -0
  86. rucio_clients-32.8.6.dist-info/licenses/AUTHORS.rst +94 -0
  87. rucio_clients-32.8.6.dist-info/licenses/LICENSE +201 -0
  88. rucio_clients-32.8.6.dist-info/top_level.txt +1 -0
@@ -0,0 +1,414 @@
1
+ # -*- coding: utf-8 -*-
2
+ # Copyright European Organization for Nuclear Research (CERN) since 2012
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+ import logging
17
+ import os
18
+ import re
19
+
20
+ from rucio.common import exception
21
+ from rucio.common.utils import execute, PREFERRED_CHECKSUM
22
+ from rucio.rse.protocols import protocol
23
+
24
+
25
+ class Default(protocol.RSEProtocol):
26
+ """ Implementing access to RSEs using the SSH protocol."""
27
+
28
+ def __init__(self, protocol_attr, rse_settings, logger=logging.log):
29
+ """ Initializes the object with information about the referred RSE.
30
+
31
+ :param props: Properties derived from the RSE Repository
32
+ """
33
+ super(Default, self).__init__(protocol_attr, rse_settings, logger=logger)
34
+
35
+ self.scheme = self.attributes['scheme']
36
+ self.hostname = self.attributes['hostname']
37
+ self.port = str(self.attributes['port'])
38
+ self.path = None
39
+ if self.attributes['extended_attributes'] is not None and\
40
+ 'user' in list(self.attributes['extended_attributes'].keys()):
41
+ self.sshuser = self.attributes['extended_attributes']['user'] + '@'
42
+ else:
43
+ self.sshuser = ''
44
+ self.logger = logger
45
+
46
+ def path2pfn(self, path):
47
+ """
48
+ Returns a fully qualified PFN for the file referred by path.
49
+
50
+ :param path: The path to the file.
51
+
52
+ :returns: Fully qualified PFN.
53
+
54
+ """
55
+ self.logger(logging.DEBUG, 'ssh.path2pfn: path: {}'.format(path))
56
+ if not path.startswith(str(self.scheme) + '://'):
57
+ return '%s://%s%s:%s/%s' % (self.scheme, self.sshuser, self.hostname, self.port, path)
58
+ else:
59
+ return path
60
+
61
+ def exists(self, pfn):
62
+ """ Checks if the requested file is known by the referred RSE.
63
+
64
+ :param pfn: Physical file name
65
+
66
+ :returns: True if the file exists, False if it doesn't
67
+
68
+ :raise ServiceUnavailable
69
+ """
70
+ self.logger(logging.DEBUG, 'ssh.exists: pfn: {}'.format(pfn))
71
+ try:
72
+ path = self.pfn2path(pfn)
73
+ cmd = 'ssh -p %s %s%s find %s' % (self.port, self.sshuser, self.hostname, path)
74
+ self.logger(logging.DEBUG, 'ssh.exists: cmd: {}'.format(cmd))
75
+ status, out, err = execute(cmd)
76
+ if status:
77
+ return False
78
+ except Exception as e:
79
+ raise exception.ServiceUnavailable(e)
80
+
81
+ return True
82
+
83
+ def stat(self, path):
84
+ """
85
+ Returns the stats of a file.
86
+
87
+ :param path: path to file
88
+
89
+ :raises ServiceUnavailable: if some generic error occured in the library.
90
+
91
+ :returns: a dict with two keys, filesize and an element of GLOBALLY_SUPPORTED_CHECKSUMS.
92
+ """
93
+ self.logger(logging.DEBUG, 'ssh.stat: path: {}'.format(path))
94
+ ret = {}
95
+ chsum = None
96
+ path = self.pfn2path(path)
97
+
98
+ try:
99
+ # ssh stat for getting filesize
100
+ cmd = 'ssh -p {0} {1}{2} stat --printf="%s" {3}'.format(self.port, self.sshuser, self.hostname, path)
101
+ self.logger(logging.DEBUG, 'ssh.stat: filesize cmd: {}'.format(cmd))
102
+ status_stat, out, err = execute(cmd)
103
+ if status_stat == 0:
104
+ ret['filesize'] = out
105
+
106
+ # ssh query checksum for getting md5 checksum
107
+ cmd = 'ssh -p %s %s%s md5sum %s' % (self.port, self.sshuser, self.hostname, path)
108
+ self.logger(logging.DEBUG, 'ssh.stat: checksum cmd: {}'.format(cmd))
109
+ status_query, out, err = execute(cmd)
110
+
111
+ if status_query == 0:
112
+ chsum = 'md5'
113
+ val = out.strip(' ').split()
114
+ ret[chsum] = val[0]
115
+
116
+ except Exception as e:
117
+ raise exception.ServiceUnavailable(e)
118
+
119
+ if 'filesize' not in ret:
120
+ raise exception.ServiceUnavailable('Filesize could not be retrieved.')
121
+ if PREFERRED_CHECKSUM != chsum or not chsum:
122
+ msg = '{} does not match with {}'.format(chsum, PREFERRED_CHECKSUM)
123
+ raise exception.RSEChecksumUnavailable(msg)
124
+
125
+ return ret
126
+
127
+ def pfn2path(self, pfn):
128
+ """
129
+ Returns the path of a file given the pfn, i.e. scheme, user and hostname are subtracted from the pfn.
130
+
131
+ :param path: pfn of a file
132
+
133
+ :returns: path.
134
+ """
135
+ path = pfn
136
+ if pfn.startswith(str(self.scheme) + '://'):
137
+ self.logger(logging.DEBUG, 'ssh.pfn2path: pfn: {}'.format(pfn))
138
+ prefix = self.attributes['prefix']
139
+ path = pfn.partition(self.attributes['prefix'])[2]
140
+ path = prefix + path
141
+ return path
142
+
143
+ def lfns2pfns(self, lfns):
144
+ """
145
+ Returns a fully qualified PFN for the file referred by path.
146
+
147
+ :param path: The path to the file.
148
+
149
+ :returns: Fully qualified PFN.
150
+ """
151
+ self.logger(logging.DEBUG, 'ssh.lfns2pfns: lfns: {}'.format(lfns))
152
+ pfns = {}
153
+ prefix = self.attributes['prefix']
154
+
155
+ if not prefix.startswith('/'):
156
+ prefix = ''.join(['/', prefix])
157
+ if not prefix.endswith('/'):
158
+ prefix = ''.join([prefix, '/'])
159
+
160
+ lfns = [lfns] if type(lfns) == dict else lfns
161
+ for lfn in lfns:
162
+ scope, name = lfn['scope'], lfn['name']
163
+ if 'path' in lfn and lfn['path'] is not None:
164
+ pfns['%s:%s' % (scope, name)] = ''.join([self.attributes['scheme'], '://', self.sshuser, self.hostname, ':', self.port, prefix, lfn['path']])
165
+ else:
166
+ pfns['%s:%s' % (scope, name)] = ''.join([self.attributes['scheme'], '://', self.sshuser, self.hostname, ':', self.port, prefix, self._get_path(scope=scope, name=name)])
167
+ return pfns
168
+
169
+ def connect(self):
170
+ """ Establishes the actual connection to the referred RSE.
171
+
172
+ :raises RSEAccessDenied
173
+ """
174
+ self.logger(logging.DEBUG, 'ssh.connect: port: {}, hostname {}, ssh-user {}'.format(self.port, self.hostname, self.sshuser))
175
+ try:
176
+ cmd = 'ssh -p %s %s%s echo ok 2>&1' % (self.port, self.sshuser, self.hostname)
177
+ status, out, err = execute(cmd)
178
+ checker = re.search(r'ok', out)
179
+ if not checker:
180
+ raise exception.RSEAccessDenied(err)
181
+ except Exception as e:
182
+ raise exception.RSEAccessDenied(e)
183
+
184
+ def close(self):
185
+ """ Closes the connection to RSE."""
186
+ pass
187
+
188
+ def get(self, pfn, dest, transfer_timeout=None):
189
+ """ Provides access to files stored inside connected the RSE.
190
+
191
+ :param pfn: Physical file name of requested file
192
+ :param dest: Name and path of the files when stored at the client
193
+ :param transfer_timeout: Transfer timeout (in seconds) - dummy
194
+
195
+ :raises DestinationNotAccessible, ServiceUnavailable, SourceNotFound
196
+ """
197
+ self.logger(logging.DEBUG, 'ssh.get: pfn: {}'.format(pfn))
198
+ try:
199
+ path = self.pfn2path(pfn)
200
+ destdir = os.path.dirname(dest)
201
+ cmd = 'mkdir -p %s' % (destdir)
202
+ self.logger(logging.DEBUG, 'ssh.get: cmd: {}'.format(cmd))
203
+ status, out, err = execute(cmd)
204
+ cmd = 'scp %s%s:%s %s' % (self.sshuser, self.hostname, path, dest)
205
+ self.logger(logging.DEBUG, 'ssh.get: cmd: {}'.format(cmd))
206
+ status, out, err = execute(cmd)
207
+ if status:
208
+ raise exception.RucioException(err)
209
+ except Exception as e:
210
+ raise exception.ServiceUnavailable(e)
211
+
212
+ def put(self, filename, target, source_dir, transfer_timeout=None):
213
+ """
214
+ Allows to store files inside the referred RSE.
215
+
216
+ :param source: path to the source file on the client file system
217
+ :param target: path to the destination file on the storage
218
+ :param source_dir: Path where the to be transferred files are stored in the local file system
219
+ :param transfer_timeout: Transfer timeout (in seconds) - dummy
220
+
221
+ :raises DestinationNotAccessible: if the destination storage was not accessible.
222
+ :raises ServiceUnavailable: if some generic error occured in the library.
223
+ :raises SourceNotFound: if the source file was not found on the referred storage.
224
+ """
225
+ self.logger(logging.DEBUG, 'ssh.put: filename: {} target: {}'.format(filename, target))
226
+ source_dir = source_dir or '.'
227
+ source_url = '%s/%s' % (source_dir, filename)
228
+ self.logger(logging.DEBUG, 'ssh.put: source url: {}'.format(source_url))
229
+
230
+ path = self.pfn2path(target)
231
+ pathdir = os.path.dirname(path)
232
+ if not os.path.exists(source_url):
233
+ raise exception.SourceNotFound()
234
+ try:
235
+ cmd = 'ssh %s%s "mkdir -p %s" && scp %s %s%s:%s' % (self.sshuser, self.hostname, pathdir, source_url, self.sshuser, self.hostname, path)
236
+ self.logger(logging.DEBUG, 'ssh.put: cmd: {}'.format(cmd))
237
+ status, out, err = execute(cmd)
238
+ if status:
239
+ raise exception.RucioException(err)
240
+ except Exception as e:
241
+ raise exception.ServiceUnavailable(e)
242
+
243
+ def delete(self, pfn):
244
+ """
245
+ Deletes a file from the connected RSE.
246
+
247
+ :param pfn: Physical file name
248
+
249
+ :raises ServiceUnavailable: if some generic error occured in the library.
250
+ :raises SourceNotFound: if the source file was not found on the referred storage.
251
+ """
252
+ self.logger(logging.DEBUG, 'ssh.delete: pfn: {}'.format(pfn))
253
+ if not self.exists(pfn):
254
+ raise exception.SourceNotFound()
255
+ try:
256
+ path = self.pfn2path(pfn)
257
+ cmd = 'ssh -p %s %s%s rm %s' % (self.port, self.sshuser, self.hostname, path)
258
+ self.logger(logging.DEBUG, 'ssh.delete: cmd: {}'.format(cmd))
259
+ status, out, err = execute(cmd)
260
+ if status != 0:
261
+ raise exception.RucioException(err)
262
+ except Exception as e:
263
+ raise exception.ServiceUnavailable(e)
264
+
265
+ def rename(self, pfn, new_pfn):
266
+ """ Allows to rename a file stored inside the connected RSE.
267
+
268
+ :param pfn: Current physical file name
269
+ :param new_pfn New physical file name
270
+ :raises DestinationNotAccessible: if the destination storage was not accessible.
271
+ :raises ServiceUnavailable: if some generic error occured in the library.
272
+ :raises SourceNotFound: if the source file was not found on the referred storage.
273
+ """
274
+ self.logger(logging.DEBUG, 'ssh.rename: pfn: {}'.format(pfn))
275
+ if not self.exists(pfn):
276
+ raise exception.SourceNotFound()
277
+ try:
278
+ path = self.pfn2path(pfn)
279
+ new_path = self.pfn2path(new_pfn)
280
+ new_dir = new_path[:new_path.rindex('/') + 1]
281
+ cmd = 'ssh -p %s %s%s "mkdir -p %s"' % (self.port, self.sshuser, self.hostname, new_dir)
282
+ self.logger(logging.DEBUG, 'ssh.rename: mkdir cmd: {}'.format(cmd))
283
+ status, out, err = execute(cmd)
284
+ cmd = 'ssh -p %s %s%s mv %s %s' % (self.port, self.sshuser, self.hostname, path, new_path)
285
+ self.logger(logging.DEBUG, 'ssh.rename: rename cmd: {}'.format(cmd))
286
+ status, out, err = execute(cmd)
287
+ if status != 0:
288
+ raise exception.RucioException(err)
289
+ except Exception as e:
290
+ raise exception.ServiceUnavailable(e)
291
+
292
+
293
+ class Rsync(Default):
294
+ """ Implementing access to RSEs using the ssh.Rsync implementation."""
295
+
296
+ def stat(self, path):
297
+ """
298
+ Returns the stats of a file.
299
+
300
+ :param path: path to file
301
+
302
+ :raises ServiceUnavailable: if some generic error occured in the library.
303
+
304
+ :returns: a dict with two keys, filesize and an element of GLOBALLY_SUPPORTED_CHECKSUMS.
305
+ """
306
+ self.logger(logging.DEBUG, 'rsync.stat: path: {}'.format(path))
307
+ ret = {}
308
+ chsum = None
309
+ path = self.pfn2path(path)
310
+
311
+ try:
312
+ # rsync stat for getting filesize
313
+ cmd = "rsync -an --size-only -e 'ssh -p {0}' --remove-source-files {1}{2}:{3}".format(self.port, self.sshuser, self.hostname, path)
314
+ self.logger(logging.DEBUG, 'rsync.stat: filesize cmd: {}'.format(cmd))
315
+ status_stat, out, err = execute(cmd)
316
+ if status_stat == 0:
317
+ sizestr = out.split(" ")[-4]
318
+ ret['filesize'] = sizestr.replace(',', '')
319
+
320
+ # rsync query checksum for getting md5 checksum
321
+ cmd = 'ssh -p %s %s%s md5sum %s' % (self.port, self.sshuser, self.hostname, path)
322
+ self.logger(logging.DEBUG, 'rsync.stat: checksum cmd: {}'.format(cmd))
323
+ status_query, out, err = execute(cmd)
324
+
325
+ if status_query == 0:
326
+ chsum = 'md5'
327
+ val = out.strip(' ').split()
328
+ ret[chsum] = val[0]
329
+
330
+ except Exception as e:
331
+ raise exception.ServiceUnavailable(e)
332
+
333
+ if 'filesize' not in ret:
334
+ raise exception.ServiceUnavailable('Filesize could not be retrieved.')
335
+ if PREFERRED_CHECKSUM != chsum or not chsum:
336
+ msg = '{} does not match with {}'.format(chsum, PREFERRED_CHECKSUM)
337
+ raise exception.RSEChecksumUnavailable(msg)
338
+
339
+ return ret
340
+
341
+ def connect(self):
342
+ """ Establishes the actual connection to the referred RSE.
343
+
344
+ :raises RSEAccessDenied
345
+ """
346
+ self.logger(logging.DEBUG, 'rsync.connect: port: {}, hostname {}, ssh-user {}'.format(self.port, self.hostname, self.sshuser))
347
+ try:
348
+ cmd = 'ssh -p %s %s%s echo ok 2>&1' % (self.port, self.sshuser, self.hostname)
349
+ status, out, err = execute(cmd)
350
+ checker = re.search(r'ok', out)
351
+ if not checker:
352
+ raise exception.RSEAccessDenied(err)
353
+ cmd = 'ssh -p %s %s%s type rsync' % (self.port, self.sshuser, self.hostname)
354
+ status, out, err = execute(cmd)
355
+ checker = re.search(r'rsync is', out)
356
+ if not checker:
357
+ raise exception.RSEAccessDenied(err)
358
+ self.path = out.split(" ")[2][:-1]
359
+
360
+ except Exception as e:
361
+ raise exception.RSEAccessDenied(e)
362
+
363
+ def get(self, pfn, dest, transfer_timeout=None):
364
+ """ Provides access to files stored inside connected the RSE.
365
+
366
+ :param pfn: Physical file name of requested file
367
+ :param dest: Name and path of the files when stored at the client
368
+ :param transfer_timeout: Transfer timeout (in seconds) - dummy
369
+
370
+ :raises DestinationNotAccessible, ServiceUnavailable, SourceNotFound
371
+ """
372
+ self.logger(logging.DEBUG, 'rsync.get: pfn: {}'.format(pfn))
373
+ try:
374
+ path = self.pfn2path(pfn)
375
+ destdir = os.path.dirname(dest)
376
+ cmd = 'mkdir -p %s && rsync -az -e "ssh -p %s" --append-verify %s%s:%s %s' % (destdir, self.port, self.sshuser, self.hostname, path, dest)
377
+ self.logger(logging.DEBUG, 'rsync.get: cmd: {}'.format(cmd))
378
+ status, out, err = execute(cmd)
379
+ if status:
380
+ raise exception.RucioException(err)
381
+ except Exception as e:
382
+ raise exception.ServiceUnavailable(e)
383
+
384
+ def put(self, filename, target, source_dir, transfer_timeout=None):
385
+ """
386
+ Allows to store files inside the referred RSE.
387
+
388
+ :param source: path to the source file on the client file system
389
+ :param target: path to the destination file on the storage
390
+ :param source_dir: Path where the to be transferred files are stored in the local file system
391
+ :param transfer_timeout: Transfer timeout (in seconds) - dummy
392
+
393
+ :raises DestinationNotAccessible: if the destination storage was not accessible.
394
+ :raises ServiceUnavailable: if some generic error occured in the library.
395
+ :raises SourceNotFound: if the source file was not found on the referred storage.
396
+ """
397
+ self.logger(logging.DEBUG, 'rsync.put: filename: {} target: {}'.format(filename, target))
398
+ source_dir = source_dir or '.'
399
+ source_url = '%s/%s' % (source_dir, filename)
400
+ self.logger(logging.DEBUG, 'rsync.put: source url: {}'.format(source_url))
401
+
402
+ path = self.pfn2path(target)
403
+ pathdir = os.path.dirname(path)
404
+ if not os.path.exists(source_url):
405
+ raise exception.SourceNotFound()
406
+
407
+ try:
408
+ cmd = 'ssh -p %s %s%s "mkdir -p %s" && rsync -az -e "ssh -p %s" --append-verify %s %s%s:%s' % (self.port, self.sshuser, self.hostname, pathdir, self.port, source_url, self.sshuser, self.hostname, path)
409
+ self.logger(logging.DEBUG, 'rsync.put: cmd: {}'.format(cmd))
410
+ status, out, err = execute(cmd)
411
+ if status:
412
+ raise exception.RucioException(err)
413
+ except Exception as e:
414
+ raise exception.ServiceUnavailable(e)
@@ -0,0 +1,207 @@
1
+ # -*- coding: utf-8 -*-
2
+ # Copyright European Organization for Nuclear Research (CERN) since 2012
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+ import logging
17
+ import os
18
+ from xml.dom import minidom
19
+
20
+ import requests
21
+
22
+ from rucio.common import exception
23
+ from rucio.common.utils import run_cmd_process
24
+ from rucio.rse.protocols import protocol
25
+
26
+
27
+ class Default(protocol.RSEProtocol):
28
+ """ Implementing access to RSEs using the local filesystem."""
29
+
30
+ def __init__(self, protocol_attr, rse_settings, logger=None):
31
+ """ Initializes the object with information about the referred RSE.
32
+
33
+ :param props: Properties derived from the RSE Repository
34
+ """
35
+ super(Default, self).__init__(protocol_attr, rse_settings, logger=logger)
36
+ self.attributes.pop('determinism_type', None)
37
+ self.files = []
38
+
39
+ def lfns2pfns(self, lfns):
40
+ """ Create fake storm:// path. Will be resolved at the get() stage later. """
41
+ pfns = {}
42
+
43
+ hostname = self.attributes['hostname']
44
+ if '://' in hostname:
45
+ hostname = hostname.split("://")[1]
46
+
47
+ prefix = self.attributes['prefix']
48
+ if not prefix.startswith('/'):
49
+ prefix = ''.join(['/', prefix])
50
+ if not prefix.endswith('/'):
51
+ prefix = ''.join([prefix, '/'])
52
+
53
+ lfns = [lfns] if isinstance(lfns, dict) else lfns
54
+ for lfn in lfns:
55
+ path = lfn['path'] if 'path' in lfn and lfn['path'] else self._get_path(scope=lfn['scope'],
56
+ name=lfn['name'])
57
+ pfns['%s:%s' % (lfn['scope'], lfn['name'])] = ''.join(['storm://', hostname, ':', str(self.attributes['port']), prefix, path])
58
+
59
+ return pfns
60
+
61
+ def path2pfn(self, path):
62
+ """
63
+ Retruns a fully qualified PFN for the file referred by path.
64
+
65
+ :param path: The path to the file.
66
+
67
+ :returns: Fully qualified PFN.
68
+
69
+ """
70
+ return ''.join([self.rse['scheme'], '://%s' % self.rse['hostname'], path])
71
+
72
+ def exists(self, pfn):
73
+ """ Checks if the requested file is known by the referred RSE.
74
+
75
+ :param pfn: Physical file name
76
+
77
+ :returns: True if the file exists, False if it doesn't
78
+
79
+ :raise ServiceUnavailable
80
+ """
81
+ raise NotImplementedError
82
+
83
+ def connect(self):
84
+ """ Establishes the actual connection to the referred RSE.
85
+
86
+ :param credentials: Provide all necessary information to establish a connection
87
+ to the referred storage system. Some is loaded from the repository inside the
88
+ RSE class and some must be provided specific for the SFTP protocol like
89
+ username, password, private_key, private_key_pass, port.
90
+ For details about possible additional parameters and details about their usage
91
+ see the pysftp.Connection() documentation.
92
+ NOTE: the host parametrer is overwritten with the value provided by the repository
93
+
94
+ :raise RSEAccessDenied
95
+ """
96
+ pass
97
+
98
+ def close(self):
99
+ """ Closes the connection to RSE."""
100
+ pass
101
+
102
+ def get(self, pfn, dest, transfer_timeout=None):
103
+ """ Provides access to files stored inside connected the RSE.
104
+
105
+ :param pfn: Physical file name of requested file
106
+ :param dest: Name and path of the files when stored at the client
107
+ :param transfer_timeout Transfer timeout (in seconds)
108
+
109
+ :raises DestinationNotAccessible, ServiceUnavailable, SourceNotFound
110
+ """
111
+
112
+ # retrieve the TURL from the webdav etag, REQUESTS
113
+ def requests_etag(pfn, timeout):
114
+ x509 = os.environ.get('X509_USER_PROXY')
115
+ pfn = 'https' + pfn[:5]
116
+ session = requests.Session()
117
+ output = session.request('PROPFIND', pfn, verify=False, timeout=timeout, cert=(x509, x509))
118
+ session.close()
119
+ return output.status_code, output.text
120
+
121
+ # retrieve the TURL from the webdav etag, DAVIX
122
+ def davix_etag(pfn, timeout):
123
+ pfn = 'davs' + pfn[5:]
124
+ cmd = 'davix-http --capath /cvmfs/atlas.cern.ch/repo/ATLASLocalRootBase/etc/grid-security-emi/certificates --cert $X509_USER_PROXY -X PROPFIND %s' % pfn
125
+ try:
126
+ rcode, output = run_cmd_process(cmd, timeout=timeout)
127
+ if rcode != 0:
128
+ if output:
129
+ raise exception.ServiceUnavailable("{}/n{}".format(str(output), cmd))
130
+ else:
131
+ raise exception.ServiceUnavailable('Error message from subprocess davix-http call is missing./n{}'.format(cmd))
132
+ except Exception as e:
133
+ raise exception.ServiceUnavailable('Could not retrieve STORM WebDAV ETag: {}/n{}'.format(str(e), cmd))
134
+ return rcode, output
135
+
136
+ # using prefix from AGIS primarily (ATLAS spec.)
137
+ target = None
138
+ try:
139
+ target = self.pfn2path(pfn)
140
+ name = pfn.split('/')[-1]
141
+ if name not in target:
142
+ target = None
143
+ except:
144
+ target = None
145
+ pass
146
+
147
+ # if AGIS setting failed
148
+ etag_meta = None
149
+ if not target:
150
+ # requests preferable
151
+ try:
152
+ rcode, etag_meta = requests_etag(pfn, 300)
153
+ except:
154
+ pass
155
+ # fallback to davix
156
+ if rcode != 207:
157
+ rcode, etag_meta = davix_etag(pfn, 300)
158
+
159
+ p_output = minidom.parseString(etag_meta)
160
+ # we need to strip off the quotation marks and the <timestamp> from the etag
161
+ # but since we can have multiple underscores, we have to rely on the uniqueness
162
+ # of the full LFN to make the split
163
+ target = p_output.getElementsByTagName('d:getetag')[0].childNodes[0].nodeValue.replace('"', '')
164
+ target_ending = '_' + target.split('_')[-1]
165
+ target = target.split(target_ending)[0]
166
+
167
+ # make the symlink
168
+ try:
169
+ os.symlink(target, dest)
170
+ self.logger(logging.INFO, 'StoRM protocol: {}->{}'.format(target, dest))
171
+ except Exception as e:
172
+ exception.ServiceUnavailable('Could not create symlink: %s for target %s' % (str(e), str(target)))
173
+
174
+ def pfn2path(self, pfn):
175
+ tmp = list(self.parse_pfns(pfn).values())[0]
176
+ return '/'.join([tmp['prefix'], tmp['path'], tmp['name']])
177
+
178
+ def put(self, source, target, source_dir=None, transfer_timeout=None):
179
+ """ Allows to store files inside the referred RSE.
180
+
181
+ :param source: Physical file name
182
+ :param target: Name of the file on the storage system e.g. with prefixed scope
183
+ :param source_dir Path where the to be transferred files are stored in the local file system
184
+ :param transfer_timeout Transfer timeout (in seconds)
185
+
186
+ :raises DestinationNotAccessible, ServiceUnavailable, SourceNotFound
187
+ """
188
+ raise NotImplementedError
189
+
190
+ def delete(self, pfn):
191
+ """ Deletes a file from the connected RSE.
192
+
193
+ :param pfn: Physical file name
194
+
195
+ :raises ServiceUnavailable, SourceNotFound
196
+ """
197
+ raise NotImplementedError
198
+
199
+ def rename(self, pfn, new_pfn):
200
+ """ Allows to rename a file stored inside the connected RSE.
201
+
202
+ :param pfn: Current physical file name
203
+ :param new_pfn New physical file name
204
+
205
+ :raises DestinationNotAccessible, ServiceUnavailable, SourceNotFound
206
+ """
207
+ raise NotImplementedError