rucio-clients 35.8.2__py3-none-any.whl

rucio/rse/protocols/ngarc.py

@@ -0,0 +1,209 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import errno
+import os
+
+from rucio.common.exception import FileAlreadyExists, ServiceUnavailable, SourceNotFound
+from rucio.rse.protocols import protocol
+
+try:
+    import arc  # pylint: disable=import-error
+except:
+    pass
+
+
+class DataPoint:
+    '''
+    Wrapper around arc.datapoint_from_url() which does not clean up DataPoints
+    when python objects are destroyed, leading to connection leaking when used
+    with gridftp. This class should be used instead of arc.datapoint_from_url().
+    It can be called like dp = DataPoint('gsiftp://...', uc); dp.h.Stat()
+    where uc is an arc.UserConfig object.
+    '''
+    def __init__(self, u, uc):
+        self.h = arc.datapoint_from_url(u, uc)
+
+    def __del__(self):
+        arc.DataPoint.__swig_destroy__(self.h)
+
+
+class Default(protocol.RSEProtocol):
+    """ Implementing access to RSEs using ARC client."""
+
+    def __init__(self, protocol_attr, rse_settings, logger=None):
+        """
+        Set up UserConfig object.
+        """
+        super(Default, self).__init__(protocol_attr, rse_settings, logger=logger)
+
+        # Arc logging to stdout, uncomment for debugging. Should use root
+        # logger level eventually.
+#         root_logger = arc.Logger_getRootLogger()
+#         stream = arc.LogStream(sys.stdout)
+#         root_logger.addDestination(stream)
+#         # Set threshold to VERBOSE or DEBUG for more information
+#         root_logger.setThreshold(arc.DEBUG)
+
+        self.cfg = arc.UserConfig()
+        try:
+            self.cfg.ProxyPath(os.environ['X509_USER_PROXY'])
+        except:
+            pass
+
+    def path2pfn(self, path):
+        """
+            Returns a fully qualified PFN for the file referred by path.
+
+            :param path: The path to the file.
+
+            :returns: Fully qualified PFN.
+
+        """
+        return ''.join([self.rse['scheme'], '://%s' % self.rse['hostname'], path])
+
+    def exists(self, pfn):
+        """ Checks if the requested file is known by the referred RSE.
+
+            :param pfn: Physical file name
+
+            :returns: True if the file exists, False if it doesn't
+
+            :raise  ServiceUnavailable
+        """
+        dp = DataPoint(str(pfn), self.cfg)
+        fileinfo = arc.FileInfo()
+
+        status = dp.h.Stat(fileinfo)
+        if not status:
+            if status.GetErrno() == errno.ENOENT:
+                return False
+            raise ServiceUnavailable(str(status))
+
+        return True
+
+    def connect(self):
+        """ Establishes the actual connection to the referred RSE.
+
+            :raise RSEAccessDenied
+        """
+        pass
+
+    def close(self):
+        """ Closes the connection to RSE."""
+        pass
+
+    def __arc_copy(self, src, dest, space_token=None, transfer_timeout=None):
+
+        # TODO set proxy path
+
+        # Convert the arguments to DataPoint objects
+        source = DataPoint(str(src), self.cfg)
+        if source.h is None:
+            raise ServiceUnavailable("Can't handle source %s" % src)
+
+        destination = DataPoint(str(dest), self.cfg)
+        if destination.h is None:
+            raise ServiceUnavailable("Can't handle destination %s" % dest)
+        if space_token:
+            destination.h.GetURL().AddOption('spacetoken', space_token)
+
+        # DataMover does the transfer
+        mover = arc.DataMover()
+        # Don't attempt to retry on error
+        mover.retry(False)
+        # Passive and insecure gridftp
+        mover.passive(True)
+        mover.secure(False)
+        # Do the transfer
+        status = mover.Transfer(source.h, destination.h, arc.FileCache(), arc.URLMap())
+
+        if not status:
+            if status.GetErrno() == errno.ENOENT:
+                raise SourceNotFound()
+            if status.GetErrno() == errno.EEXIST:
+                raise FileAlreadyExists()
+            raise ServiceUnavailable(str(status))
+
+    def get(self, pfn, dest, transfer_timeout=None):
+        """ Provides access to files stored inside connected the RSE.
+
+            :param pfn: Physical file name of requested file
+            :param dest: Name and path of the files when stored at the client
+            :param transfer_timeout Transfer timeout (in seconds) - dummy
+
+            :raises DestinationNotAccessible, ServiceUnavailable, SourceNotFound
+        """
+        self.__arc_copy(pfn, dest, transfer_timeout=transfer_timeout)
+
+    def put(self, source, target, source_dir=None, transfer_timeout=None):
+        """ Allows to store files inside the referred RSE.
+
+            :param source: Physical file name
+            :param target: Name of the file on the storage system e.g. with prefixed scope
+            :param source_dir Path where the to be transferred files are stored in the local file system
+            :param transfer_timeout Transfer timeout (in seconds) - dummy
+
+            :raises DestinationNotAccessible, ServiceUnavailable, SourceNotFound
+        """
+
+        if source_dir:
+            sf = source_dir + '/' + source
+        else:
+            sf = source
+
+        space_token = None
+        if self.attributes['extended_attributes'] is not None and 'space_token' in list(self.attributes['extended_attributes'].keys()):
+            space_token = self.attributes['extended_attributes']['space_token']
+
+        self.__arc_copy(sf, target, space_token, transfer_timeout=transfer_timeout)
+
+    def delete(self, pfn):
+        """ Deletes a file from the connected RSE.
+
+            :param pfn: Physical file name
+
+            :raises ServiceUnavailable, SourceNotFound
+        """
+        dp = DataPoint(str(pfn), self.cfg)
+        if dp.h is None:
+            raise ServiceUnavailable("Can't handle pfn %s" % pfn)
+
+        status = dp.h.Remove()
+        if not status:
+            if status.GetErrno() == errno.ENOENT:
+                raise SourceNotFound()
+            raise ServiceUnavailable(str(status))
+
+    def rename(self, pfn, new_pfn):
+        """ Allows to rename a file stored inside the connected RSE.
+
+            :param pfn:      Current physical file name
+            :param new_pfn  New physical file name
+
+            :raises DestinationNotAccessible, ServiceUnavailable, SourceNotFound
+        """
+        dp = DataPoint(str(pfn), self.cfg)
+        if dp.h is None:
+            raise ServiceUnavailable("Can't handle pfn %s" % pfn)
+
+        url = arc.URL(str(new_pfn))
+        if not url:
+            raise ServiceUnavailable("Can't handle new pfn %s" % new_pfn)
+
+        status = dp.h.Rename(url)
+        if not status:
+            if status.GetErrno() == errno.ENOENT:
+                raise SourceNotFound()
+            raise ServiceUnavailable(str(status))

rucio/rse/protocols/posix.py

@@ -0,0 +1,250 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import os
+import os.path
+import shutil
+from subprocess import call
+
+from rucio.common import exception
+from rucio.common.utils import adler32
+from rucio.rse.protocols import protocol
+
+
+class Default(protocol.RSEProtocol):
+    """ Implementing access to RSEs using the local filesystem."""
+
+    def exists(self, pfn):
+        """
+            Checks if the requested file is known by the referred RSE.
+
+            :param pfn: Physical file name
+
+            :returns: True if the file exists, False if it doesn't
+
+            :raises SourceNotFound: if the source file was not found on the referred storage.
+        """
+        status = ''
+        try:
+            status = os.path.exists(self.pfn2path(pfn))
+        except Exception as e:
+            raise exception.ServiceUnavailable(e)
+        return status
+
+    def connect(self):
+        """
+            Establishes the actual connection to the referred RSE.
+
+            :param credentials: needed to establish a connection with the storage.
+
+            :raises RSEAccessDenied: if no connection could be established.
+        """
+        pass
+
+    def close(self):
+        """ Closes the connection to RSE."""
+        pass
+
+    def get(self, pfn, dest, transfer_timeout=None):
+        """ Provides access to files stored inside connected the RSE.
+
+            :param pfn: Physical file name of requested file
+            :param dest: Name and path of the files when stored at the client
+            :param transfer_timeout Transfer timeout (in seconds) - dummy
+
+            :raises DestinationNotAccessible: if the destination storage was not accessible.
+            :raises ServiceUnavailable: if some generic error occurred in the library.
+            :raises SourceNotFound: if the source file was not found on the referred storage.
+         """
+        try:
+            shutil.copy(self.pfn2path(pfn), dest)
+        except OSError as e:
+            try:  # To check if the error happened local or remote
+                with open(dest, 'wb'):
+                    pass
+                call(['rm', '-rf', dest])  # noqa: S607
+            except OSError as e:
+                if e.errno == 2:
+                    raise exception.DestinationNotAccessible(e)
+                else:
+                    raise exception.ServiceUnavailable(e)
+            if e.errno == 2:
+                raise exception.SourceNotFound(e)
+            else:
+                raise exception.ServiceUnavailable(e)
+
+    def put(self, source, target, source_dir=None, transfer_timeout=None):
+        """
+            Allows to store files inside the referred RSE.
+
+            :param source: path to the source file on the client file system
+            :param target: path to the destination file on the storage
+            :param source_dir: Path where the to be transferred files are stored in the local file system
+            :param transfer_timeout Transfer timeout (in seconds) - dummy
+
+            :raises DestinationNotAccessible: if the destination storage was not accessible.
+            :raises ServiceUnavailable: if some generic error occurred in the library.
+            :raises SourceNotFound: if the source file was not found on the referred storage.
+        """
+        target = self.pfn2path(target)
+
+        if source_dir:
+            sf = source_dir + '/' + source
+        else:
+            sf = source
+        try:
+            dirs = os.path.dirname(target)
+            if not os.path.exists(dirs):
+                os.makedirs(dirs)
+            shutil.copy(sf, target)
+        except OSError as e:
+            if e.errno == 2:
+                raise exception.SourceNotFound(e)
+            elif not self.exists(self.rse['prefix']):
+                path = ''
+                for p in self.rse['prefix'].split('/'):
+                    path += p + '/'
+                    os.mkdir(path)
+                shutil.copy(sf, self.pfn2path(target))
+            else:
+                raise exception.DestinationNotAccessible(e)
+
+    def delete(self, pfn):
+        """ Deletes a file from the connected RSE.
+
+            :param pfn: pfn to the to be deleted file
+
+            :raises ServiceUnavailable: if some generic error occurred in the library.
+            :raises SourceNotFound: if the source file was not found on the referred storage.
+        """
+        try:
+            os.remove(self.pfn2path(pfn))
+        except OSError as e:
+            if e.errno == 2:
+                raise exception.SourceNotFound(e)
+
+    def rename(self, pfn, new_pfn):
+        """ Allows to rename a file stored inside the connected RSE.
+
+            :param path: path to the current file on the storage
+            :param new_path: path to the new file on the storage
+
+            :raises DestinationNotAccessible: if the destination storage was not accessible.
+            :raises ServiceUnavailable: if some generic error occurred in the library.
+            :raises SourceNotFound: if the source file was not found on the referred storage.
+        """
+        path = self.pfn2path(pfn)
+        new_path = self.pfn2path(new_pfn)
+        try:
+            if not os.path.exists(os.path.dirname(new_path)):
+                os.makedirs(os.path.dirname(new_path))
+            os.rename(path, new_path)
+        except OSError as e:
+            if e.errno == 2:
+                if self.exists(self.pfn2path(path)):
+                    raise exception.SourceNotFound(e)
+                else:
+                    raise exception.DestinationNotAccessible(e)
+            else:
+                raise exception.ServiceUnavailable(e)
+
+    def lfns2pfns(self, lfns):
+        """ Returns fully qualified PFNs for the file referred by each lfn in
+            the lfns list.
+
+            :param lfns: List of lfns. If lfn['path'] is present it is used as
+                   the path to the file, otherwise the path is constructed
+                   deterministically.
+
+            :returns: Fully qualified PFNs.
+        """
+        pfns = {}
+        prefix = self.attributes['prefix']
+
+        if not prefix.startswith('/'):
+            prefix = ''.join(['/', prefix])
+        if not prefix.endswith('/'):
+            prefix = ''.join([prefix, '/'])
+
+        lfns = [lfns] if isinstance(lfns, dict) else lfns
+        for lfn in lfns:
+            scope, name = str(lfn['scope']), lfn['name']
+            if lfn.get('path'):
+                pfns['%s:%s' % (scope, name)] = ''.join([self.attributes['scheme'],
+                                                         '://',
+                                                         self.attributes['hostname'],
+                                                         prefix,
+                                                         lfn['path'] if not lfn['path'].startswith('/') else lfn['path'][1:]
+                                                         ])
+            else:
+                pfns['%s:%s' % (scope, name)] = ''.join([self.attributes['scheme'],
+                                                         '://',
+                                                         self.attributes['hostname'],
+                                                         prefix,
+                                                         self._get_path(scope=scope, name=name)
+                                                         ])
+        return pfns
+
+    def pfn2path(self, pfn):
+        tmp = list(self.parse_pfns(pfn).values())[0]
+        return '/'.join([tmp['prefix'], tmp['path'], tmp['name']])
+
+    def stat(self, pfn):
+        """ Determines the file size in bytes and checksum (adler32) of the provided file.
+
+            :param pfn: The PFN the file.
+
+            :returns: a dict containing the keys filesize and adler32.
+        """
+        path = self.pfn2path(pfn)
+        return {'filesize': os.stat(path)[os.path.stat.ST_SIZE], 'adler32': adler32(path)}
+
+
+class Symlink(Default):
+    """ Implementing access to RSEs using the local filesystem, creating a symlink on a get """
+
+    def get(self, pfn, dest, transfer_timeout=None):
+        """ Provides access to files stored inside connected the RSE.
+            A download/get will create a symlink on the local file system pointing to the
+            underlying file. Other operations act directly on the remote file.
+            :param pfn: Physical file name of requested file
+            :param dest: Name and path of the files when stored at the client
+            :param transfer_timeout Transfer timeout (in seconds) - dummy
+            :raises DestinationNotAccessible: if the destination storage was not accessible.
+            :raises ServiceUnavailable: if some generic error occurred in the library.
+            :raises SourceNotFound: if the source file was not found on the referred storage.
+         """
+        path = self.pfn2path(pfn)
+        os.symlink(path, dest)
+        self.logger(logging.DEBUG,
+                    'Symlink {} created for {} from {}'
+                    .format(dest, path, pfn))
+        if not os.lstat(dest):
+            # problem in creating the symlink
+            self.logger(logging.ERROR, 'Symlink {} could not be created'.format(dest))
+            raise exception.DestinationNotAccessible()
+        if not os.path.exists(dest):
+            # could not find the file following the symlink
+            self.logger(logging.ERROR, 'Symlink {} appears to be a broken link to {}'
+                        .format(dest, path))
+            if os.lstat(dest) and os.path.islink(dest):
+                os.unlink(dest)
+            raise exception.SourceNotFound()
+
+    def pfn2path(self, pfn):
+        # obtain path and sanitise from multiple slashes, etc
+        path = os.path.normpath(super().pfn2path(pfn))
+        self.logger(logging.DEBUG, 'Extracted path: {} from: {}'.format(path, pfn))
+        return path