rucio-clients 32.8.6__py3-none-any.whl → 35.8.0__py3-none-any.whl

rucio/rse/__init__.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -47,8 +46,8 @@ def get_signed_url_server(rse, service, op, url, vo='def'):
     '''
     get_signed_url_server
     '''
-    from rucio.core.rse import get_rse_id
     from rucio.core.credential import get_signed_url
+    from rucio.core.rse import get_rse_id
 
     rse_id = get_rse_id(rse=rse, vo=vo)
     return get_signed_url(rse_id, service, op, url)
@@ -78,9 +77,9 @@ if rsemanager.CLIENT_MODE:   # pylint:disable=no-member
 
 
 if rsemanager.SERVER_MODE:   # pylint:disable=no-member
-    from rucio.core.rse import get_rse_protocols, get_rse_id
-    from rucio.core.vo import map_vo
     from rucio.common.cache import make_region_memcached
+    from rucio.core.rse import get_rse_id, get_rse_protocols
+    from rucio.core.vo import map_vo
 
     def tmp_rse_info(rse=None, vo='def', rse_id=None, session=None):
         if rse_id is None:

rucio/rse/protocols/__init__.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

rucio/rse/protocols/bittorrent.py

@@ -0,0 +1,184 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import base64
+import logging
+import os.path
+import time
+from urllib.parse import parse_qs, urlencode, urlparse
+
+from rucio.common import exception
+from rucio.common.extra import import_extras
+from rucio.common.utils import construct_torrent, resolve_ip
+from rucio.rse import rsemanager
+from rucio.rse.protocols.protocol import RSEProtocol
+
+EXTRA_MODULES = import_extras(['libtorrent'])
+
+lt = None
+if EXTRA_MODULES['libtorrent']:
+    import libtorrent as lt  # pylint: disable=E0401
+
+if getattr(rsemanager, 'CLIENT_MODE', None):
+    from rucio.client.didclient import DIDClient
+
+    def _fetch_meta_client(rse_id: str, scope: str, name: str):
+        return DIDClient().get_metadata(scope=scope, name=name, plugin='all')
+
+    _fetch_meta = _fetch_meta_client
+else:
+    from rucio.common.types import InternalScope
+    from rucio.core.did import get_metadata
+    from rucio.core.rse import get_rse_vo
+
+    def _fetch_meta_server(rse_id: str, scope: str, name: str):
+        vo = get_rse_vo(rse_id)
+        return get_metadata(scope=InternalScope(scope, vo=vo), name=name, plugin='all')
+
+    _fetch_meta = _fetch_meta_server
+
+
+class Default(RSEProtocol):
+
+    def __init__(self, protocol_attr, rse_settings, logger=logging.log):
+        super(Default, self).__init__(protocol_attr, rse_settings, logger=logger)
+        self.logger = logger
+
+    def lfns2pfns(self, lfns):
+        pfns = {}
+        prefix = self.attributes['prefix']
+
+        if not prefix.startswith('/'):
+            prefix = ''.join(['/', prefix])
+        if not prefix.endswith('/'):
+            prefix = ''.join([prefix, '/'])
+
+        host_port = '%s:%s' % (self.attributes['hostname'], str(self.attributes['port']))
+
+        lfns = [lfns] if isinstance(lfns, dict) else lfns
+        for lfn in lfns:
+            scope, name = lfn['scope'], lfn['name']
+
+            if 'path' in lfn and lfn['path'] is not None:
+                path = lfn['path'] if not lfn['path'].startswith('/') else lfn['path'][1:]
+            else:
+                path = self._get_path(scope=scope, name=name)
+
+            scope_name = '%s:%s' % (scope, name)
+
+            query = {
+                'x.pe': host_port,
+                'x.rucio_scope': scope,
+                'x.rucio_name': name,
+                'x.rucio_path': ''.join((prefix, path))
+            }
+            pfns[scope_name] = 'magnet:?' + urlencode(query)
+
+        return pfns
+
+    def parse_pfns(self, pfns):
+        ret = dict()
+        pfns = [pfns] if isinstance(pfns, str) else pfns
+
+        for pfn in pfns:
+            parsed = urlparse(pfn)
+            scheme = parsed.scheme
+
+            query = parse_qs(parsed.query)
+            host_port = next(iter(query.get('x.pe', [])), ':')
+            hostname, port = host_port.split(':')
+            port = int(port)
+            path = next(iter(query.get('x.rucio_path', [])), '')
+            scope = next(iter(query.get('x.rucio_scope', [])), '')
+            name = next(iter(query.get('x.rucio_name', [])), '')
+
+            # Protect against 'lazy' defined prefixes for RSEs in the repository
+            if not self.attributes['prefix'].startswith('/'):
+                self.attributes['prefix'] = '/' + self.attributes['prefix']
+            if not self.attributes['prefix'].endswith('/'):
+                self.attributes['prefix'] += '/'
+
+            if self.attributes['hostname'] != hostname:
+                if self.attributes['hostname'] != 'localhost':  # In the database empty hostnames are replaced with localhost but for some URIs (e.g. file) a hostname is not included
+                    raise exception.RSEFileNameNotSupported('Invalid hostname: provided \'%s\', expected \'%s\'' % (hostname, self.attributes['hostname']))
+
+            if self.attributes['port'] != port:
+                raise exception.RSEFileNameNotSupported('Invalid port: provided \'%s\', expected \'%s\'' % (port, self.attributes['port']))
+
+            if not path.startswith(self.attributes['prefix']):
+                raise exception.RSEFileNameNotSupported('Invalid prefix: provided \'%s\', expected \'%s\'' % ('/'.join(path.split('/')[0:len(self.attributes['prefix'].split('/')) - 1]),
+                                                                                                              self.attributes['prefix']))  # len(...)-1 due to the leading '/
+
+            # Splitting parsed.path into prefix, path, filename
+            prefix = self.attributes['prefix']
+            path = path.partition(self.attributes['prefix'])[2]
+            path = '/'.join(path.split('/')[:-1])
+            if not path.startswith('/'):
+                path = '/' + path
+            if path != '/' and not path.endswith('/'):
+                path = path + '/'
+            ret[pfn] = {'path': path, 'scope': scope, 'name': name, 'scheme': scheme, 'prefix': prefix, 'port': port, 'hostname': hostname, }
+
+        return ret
+
+    def connect(self):
+        pass
+
+    def close(self):
+        pass
+
+    def get(self, path, dest, transfer_timeout=None):
+        if not lt:
+            raise exception.RucioException('The libtorrent python package is required to perform this operation')
+
+        [lfn] = self.parse_pfns([path]).values()
+        scope = lfn['scope']
+        name = lfn['name']
+        hostname = lfn['hostname']
+        port = lfn['port']
+
+        meta = _fetch_meta(rse_id=self.rse['id'], scope=scope, name=name)
+        pieces_root = base64.b64decode(meta.get('bittorrent_pieces_root', ''))
+        if not pieces_root:
+            raise exception.RucioException('Torrent metadata missing. Cannot download file.')
+
+        length = meta.get('bytes')
+        piece_length = meta.get('bittorrent_piece_length', 0)
+        pieces_layers = base64.b64decode(meta.get('bittorrent_pieces_layers', ''))
+
+        _, torrent = construct_torrent(
+            scope=scope,
+            name=name,
+            length=length,
+            piece_length=piece_length,
+            pieces_root=pieces_root,
+            pieces_layers=pieces_layers,
+        )
+
+        ses = lt.session()  # type: ignore # noqa
+        params = {
+            'ti': lt.torrent_info(torrent),  # type: ignore # noqa
+            'save_path': os.path.dirname(dest),
+            'name': os.path.basename(dest),
+            'renamed_files': {0: os.path.basename(dest)},
+        }
+
+        handle = ses.add_torrent(params)
+        try:
+            handle.resume()
+            handle.connect_peer((resolve_ip(hostname), port))
+            while handle.status().progress != 1.0:
+                time.sleep(0.25)
+        finally:
+            ses.remove_torrent(handle)

rucio/rse/protocols/cache.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -41,7 +40,7 @@ class Default(protocol.RSEProtocol):
 
     def path2pfn(self, path):
         """
-            Retruns a fully qualified PFN for the file referred by path.
+            Returns a fully qualified PFN for the file referred by path.
 
             :param path: The path to the file.
 

rucio/rse/protocols/dummy.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -30,7 +29,7 @@ class Default(protocol.RSEProtocol):
 
     def path2pfn(self, path):
         """
-            Retruns a fully qualified PFN for the file referred by path.
+            Returns a fully qualified PFN for the file referred by path.
 
             :param path: The path to the file.
 

rucio/rse/protocols/gfal.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -22,7 +21,7 @@ import subprocess
 import urllib.parse as urlparse
 from threading import Timer
 
-from rucio.common import exception, config
+from rucio.common import config, exception
 from rucio.common.constraints import STRING_TYPES
 from rucio.common.utils import GLOBALLY_SUPPORTED_CHECKSUMS, PREFERRED_CHECKSUM
 from rucio.rse.protocols import protocol
@@ -51,7 +50,7 @@ class Default(protocol.RSEProtocol):
 
         :returns: Fully qualified PFN.
         """
-        lfns = [lfns] if type(lfns) == dict else lfns
+        lfns = [lfns] if isinstance(lfns, dict) else lfns
 
         pfns = {}
         prefix = self.attributes['prefix']
@@ -126,7 +125,7 @@ class Default(protocol.RSEProtocol):
             if not path.startswith(self.attributes['prefix']):
                 raise exception.RSEFileNameNotSupported('Invalid prefix: provided \'%s\', expected \'%s\'' % ('/'.join(path.split('/')[0:len(self.attributes['prefix'].split('/')) - 1]),
                                                                                                               self.attributes['prefix']))  # len(...)-1 due to the leading '/
-            # Spliting path into prefix, path, filename
+            # Splitting path into prefix, path, filename
             prefix = self.attributes['prefix']
             path = path.partition(self.attributes['prefix'])[2]
             name = path.split('/')[-1]
@@ -231,7 +230,7 @@ class Default(protocol.RSEProtocol):
         :param transfer_timeout: Transfer timeout (in seconds)
 
         :raises DestinationNotAccessible: if the destination storage was not accessible.
-        :raises ServiceUnavailable: if some generic error occured in the library.
+        :raises ServiceUnavailable: if some generic error occurred in the library.
         :raises SourceNotFound: if the source file was not found on the referred storage.
         """
         self.logger(logging.DEBUG, 'downloading file from {} to {}'.format(path, dest))
@@ -261,7 +260,7 @@ class Default(protocol.RSEProtocol):
         :param transfer_timeout: Transfer timeout (in seconds)
 
         :raises DestinationNotAccessible: if the destination storage was not accessible.
-        :raises ServiceUnavailable: if some generic error occured in the library.
+        :raises ServiceUnavailable: if some generic error occurred in the library.
         :raises SourceNotFound: if the source file was not found on the referred storage.
         """
         self.logger(logging.DEBUG, 'uploading file from {} to {}'.format(source, target))
@@ -294,7 +293,7 @@ class Default(protocol.RSEProtocol):
 
         :param path: path to the to be deleted file
 
-        :raises ServiceUnavailable: if some generic error occured in the library.
+        :raises ServiceUnavailable: if some generic error occurred in the library.
         :raises SourceNotFound: if the source file was not found on the referred storage.
         """
         self.logger(logging.DEBUG, 'deleting file {}'.format(path))
@@ -318,7 +317,7 @@ class Default(protocol.RSEProtocol):
         :param new_path: path to the new file on the storage
 
         :raises DestinationNotAccessible: if the destination storage was not accessible.
-        :raises ServiceUnavailable: if some generic error occured in the library.
+        :raises ServiceUnavailable: if some generic error occurred in the library.
         :raises SourceNotFound: if the source file was not found on the referred storage.
         """
         self.logger(logging.DEBUG, 'renaming file from {} to {}'.format(path, new_path))
@@ -370,7 +369,7 @@ class Default(protocol.RSEProtocol):
 
             :param path: path to file
 
-            :raises ServiceUnavailable: if some generic error occured in the library.
+            :raises ServiceUnavailable: if some generic error occurred in the library.
 
             :returns: a dict with two keys, filesize and an element of GLOBALLY_SUPPORTED_CHECKSUMS.
         """
@@ -394,6 +393,9 @@ class Default(protocol.RSEProtocol):
 
         ret['filesize'] = stats[7]
 
+        if not self.rse.get('verify_checksum', True):
+            return ret
+
         message = "\n"
         try:
             ret[PREFERRED_CHECKSUM] = ctx.checksum(path, str(PREFERRED_CHECKSUM.upper()))
@@ -551,7 +553,7 @@ class Default(protocol.RSEProtocol):
 
         :returns: a list with dict containing 'totalsize' and 'unusedsize'
 
-        :raises ServiceUnavailable: if some generic error occured in the library.
+        :raises ServiceUnavailable: if some generic error occurred in the library.
         """
         endpoint_basepath = self.path2pfn(self.attributes['prefix'])
         self.logger(logging.DEBUG, 'getting space usage from {}'.format(endpoint_basepath))

rucio/rse/protocols/globus.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -17,10 +16,11 @@ import logging
 from urllib.parse import urlparse
 
 from rucio.common import exception
+from rucio.common.constants import RseAttr
 from rucio.common.extra import import_extras
 from rucio.core.rse import get_rse_attribute
 from rucio.rse.protocols.protocol import RSEProtocol
-from rucio.transfertool.globus_library import get_transfer_client, send_delete_task, send_bulk_delete_task
+from rucio.transfertool.globus_library import get_transfer_client, send_bulk_delete_task, send_delete_task
 
 EXTRA_MODULES = import_extras(['globus_sdk'])
 
@@ -37,12 +37,12 @@ class GlobusRSEProtocol(RSEProtocol):
             :param props: Properties of the requested protocol
         """
         super(GlobusRSEProtocol, self).__init__(protocol_attr, rse_settings, logger=logger)
-        self.globus_endpoint_id = get_rse_attribute(self.rse.get('id'), 'globus_endpoint_id')
+        self.globus_endpoint_id = get_rse_attribute(self.rse.get('id'), RseAttr.GLOBUS_ENDPOINT_ID)
         self.logger = logger
 
     def lfns2pfns(self, lfns):
         """
-            Retruns a fully qualified PFN for the file referred by path.
+            Returns a fully qualified PFN for the file referred by path.
 
             :param path: The path to the file.
 
@@ -80,7 +80,7 @@ class GlobusRSEProtocol(RSEProtocol):
 
     def parse_pfns(self, pfns):
         """
-            Splits the given PFN into the parts known by the protocol. It is also checked if the provided protocol supportes the given PFNs.
+            Splits the given PFN into the parts known by the protocol. It is also checked if the provided protocol supports the given PFNs.
 
             :param pfns: a list of a fully qualified PFNs
 
@@ -117,7 +117,7 @@ class GlobusRSEProtocol(RSEProtocol):
                 raise exception.RSEFileNameNotSupported('Invalid prefix: provided \'%s\', expected \'%s\'' % ('/'.join(path.split('/')[0:len(self.attributes['prefix'].split('/')) - 1]),
                                                                                                               self.attributes['prefix']))  # len(...)-1 due to the leading '/
 
-            # Spliting parsed.path into prefix, path, filename
+            # Splitting parsed.path into prefix, path, filename
             prefix = self.attributes['prefix']
             path = path.partition(self.attributes['prefix'])[2]
             name = path.split('/')[-1]
@@ -190,7 +190,7 @@ class GlobusRSEProtocol(RSEProtocol):
 
             :param path: path to the to be deleted file
 
-            :raises ServiceUnavailable: if some generic error occured in the library.
+            :raises ServiceUnavailable: if some generic error occurred in the library.
             :raises SourceNotFound: if the source file was not found on the referred storage.
         """
         if self.globus_endpoint_id:

rucio/rse/protocols/gsiftp.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -53,12 +52,12 @@ class Default(protocol.RSEProtocol):
 
         :returns: a list with dict containing 'totalsize' and 'unusedsize'
 
-        :raises ServiceUnavailable: if some generic error occured in the library.
+        :raises ServiceUnavailable: if some generic error occurred in the library.
         """
         rse_name = self.rse['rse']
         dest = '/tmp/rucio-gsiftp-site-size_' + rse_name
         space_usage_url = ''
-        # url of space usage json, woud be nicer to have it in rse_settings
+        # url of space usage json, would be nicer to have it in rse_settings
         agis = requests.get('http://atlas-agis-api.cern.ch/request/ddmendpoint/query/list/?json').json()
         agis_token = ''
         for res in agis:

rucio/rse/protocols/http_cache.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -42,7 +41,7 @@ class Default(ngarc.Default):
 
     def path2pfn(self, path):
         """
-            Retruns a fully qualified PFN for the file referred by path.
+            Returns a fully qualified PFN for the file referred by path.
 
             :param path: The path to the file.
 

rucio/rse/protocols/mock.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -31,7 +30,7 @@ class Default(protocol.RSEProtocol):
 
     def path2pfn(self, path):
         """
-            Retruns a fully qualified PFN for the file referred by path.
+            Returns a fully qualified PFN for the file referred by path.
 
             :param path: The path to the file.
 

rucio/rse/protocols/ngarc.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -65,7 +64,7 @@ class Default(protocol.RSEProtocol):
 
     def path2pfn(self, path):
         """
-            Retruns a fully qualified PFN for the file referred by path.
+            Returns a fully qualified PFN for the file referred by path.
 
             :param path: The path to the file.
 

rucio/rse/protocols/posix.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -48,7 +47,7 @@ class Default(protocol.RSEProtocol):
         """
             Establishes the actual connection to the referred RSE.
 
-            :param credentials: needed to establish a connection with the stroage.
+            :param credentials: needed to establish a connection with the storage.
 
             :raises RSEAccessDenied: if no connection could be established.
         """
@@ -66,17 +65,17 @@ class Default(protocol.RSEProtocol):
             :param transfer_timeout Transfer timeout (in seconds) - dummy
 
             :raises DestinationNotAccessible: if the destination storage was not accessible.
-            :raises ServiceUnavailable: if some generic error occured in the library.
+            :raises ServiceUnavailable: if some generic error occurred in the library.
             :raises SourceNotFound: if the source file was not found on the referred storage.
          """
         try:
             shutil.copy(self.pfn2path(pfn), dest)
-        except IOError as e:
-            try:  # To check if the error happend local or remote
+        except OSError as e:
+            try:  # To check if the error happened local or remote
                 with open(dest, 'wb'):
                     pass
-                call(['rm', '-rf', dest])
-            except IOError as e:
+                call(['rm', '-rf', dest])  # noqa: S607
+            except OSError as e:
                 if e.errno == 2:
                     raise exception.DestinationNotAccessible(e)
                 else:
@@ -96,7 +95,7 @@ class Default(protocol.RSEProtocol):
             :param transfer_timeout Transfer timeout (in seconds) - dummy
 
             :raises DestinationNotAccessible: if the destination storage was not accessible.
-            :raises ServiceUnavailable: if some generic error occured in the library.
+            :raises ServiceUnavailable: if some generic error occurred in the library.
             :raises SourceNotFound: if the source file was not found on the referred storage.
         """
         target = self.pfn2path(target)
@@ -110,7 +109,7 @@ class Default(protocol.RSEProtocol):
             if not os.path.exists(dirs):
                 os.makedirs(dirs)
             shutil.copy(sf, target)
-        except IOError as e:
+        except OSError as e:
             if e.errno == 2:
                 raise exception.SourceNotFound(e)
             elif not self.exists(self.rse['prefix']):
@@ -127,7 +126,7 @@ class Default(protocol.RSEProtocol):
 
             :param pfn: pfn to the to be deleted file
 
-            :raises ServiceUnavailable: if some generic error occured in the library.
+            :raises ServiceUnavailable: if some generic error occurred in the library.
             :raises SourceNotFound: if the source file was not found on the referred storage.
         """
         try:
@@ -143,7 +142,7 @@ class Default(protocol.RSEProtocol):
             :param new_path: path to the new file on the storage
 
             :raises DestinationNotAccessible: if the destination storage was not accessible.
-            :raises ServiceUnavailable: if some generic error occured in the library.
+            :raises ServiceUnavailable: if some generic error occurred in the library.
             :raises SourceNotFound: if the source file was not found on the referred storage.
         """
         path = self.pfn2path(pfn)
@@ -152,7 +151,7 @@ class Default(protocol.RSEProtocol):
             if not os.path.exists(os.path.dirname(new_path)):
                 os.makedirs(os.path.dirname(new_path))
             os.rename(path, new_path)
-        except IOError as e:
+        except OSError as e:
             if e.errno == 2:
                 if self.exists(self.pfn2path(path)):
                     raise exception.SourceNotFound(e)
@@ -224,7 +223,7 @@ class Symlink(Default):
             :param dest: Name and path of the files when stored at the client
             :param transfer_timeout Transfer timeout (in seconds) - dummy
             :raises DestinationNotAccessible: if the destination storage was not accessible.
-            :raises ServiceUnavailable: if some generic error occured in the library.
+            :raises ServiceUnavailable: if some generic error occurred in the library.
             :raises SourceNotFound: if the source file was not found on the referred storage.
          """
         path = self.pfn2path(pfn)