rucio-clients 35.7.0__py3-none-any.whl → 37.0.0rc2__py3-none-any.whl

rucio/rse/protocols/protocol.py

@@ -16,326 +16,50 @@
 This module defines the base class for implementing a transfer protocol,
 along with some of the default methods for LFN2PFN translations.
 """
-import hashlib
 import logging
-from collections.abc import Callable, Mapping
-from configparser import NoOptionError, NoSectionError
-from typing import Any, Optional, TypeVar
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING, Any, Optional, Union
 from urllib.parse import urlparse
 
-from rucio.common import config, exception
-from rucio.common.constants import RseAttr
-from rucio.common.plugins import PolicyPackageAlgorithms
+from rucio.common import exception
 from rucio.rse import rsemanager
+from rucio.rse.translation import RSEDeterministicTranslation
 
 if getattr(rsemanager, 'CLIENT_MODE', None):
     from rucio.client.rseclient import RSEClient
 
 if getattr(rsemanager, 'SERVER_MODE', None):
-    from rucio.common.types import InternalScope
     from rucio.core import replica
     from rucio.core.rse import get_rse_vo
 
+if getattr(rsemanager, 'SERVER_MODE', None) or TYPE_CHECKING:
+    from rucio.common.types import InternalScope, LFNDict, LoggerFunction, RSESettingsDict
 
-class RSEDeterministicScopeTranslation(PolicyPackageAlgorithms):
-    """
-        Translates a pfn dictionary into a scope and name
-    """
+if TYPE_CHECKING:
+    from collections.abc import Iterable
 
-    _algorithm_type = "pfn2lfn"
+    from rucio.common.types import DIDDict
 
-    def __init__(self, vo: str = 'def'):
-        super().__init__()
 
-        self.register(RSEDeterministicScopeTranslation._default, "def")
-        self.register(RSEDeterministicScopeTranslation._atlas, "atlas")
+class RSEProtocol(ABC):
+    """ This class is virtual and acts as a base to inherit new protocols from. It further provides some common functionality which applies for the majority of the protocols."""
 
-        logger = logging.getLogger(__name__)
-
-        try:
-            # Use the function defined in the policy package if it's configured so
-            algorithm_name = config.config_get('policy', self._algorithm_type)
-        except (NoOptionError, NoSectionError, RuntimeError):
-            # Don't use a function from the policy package. Use one defined in this class according to vo
-            logger.debug("PFN2LFN function will not be fetched from the policy package")
-            if super()._supports(self._algorithm_type, vo):
-                algorithm_name = vo
-            else:
-                algorithm_name = "def"
-
-        self.parser = self.get_parser(algorithm_name)
-
-    @classmethod
-    def get_parser(cls, algorithm_name: str) -> Callable[..., Any]:
-        return super()._get_one_algorithm(cls._algorithm_type, algorithm_name)
-
-    @classmethod
-    def register(
-        cls,
-        pfn2lfn_callable: Callable,
-        name: Optional[str] = None
-    ) -> None:
-        """
-        Provided a callable function, register it as one of the valid PFN2LFN algorithms.
-
-
-        :param pfn2lfn_callable: Callable function to use.
-        :param name: Algorithm name used for registration.
-        """
-        if name is None:
-            name = pfn2lfn_callable.__name__
-        algorithm_dict = {name: pfn2lfn_callable}
-        super()._register(cls._algorithm_type, algorithm_dict)
-
-    @staticmethod
-    def _default(parsed_pfn: Mapping[str, str]) -> tuple[str, str]:
-        """ Translate pfn to name/scope pair
-
-        :param parsed_pfn: dictionary representing pfn containing:
-            - path: str,
-            - name: str
-        :return: tuple containing name, scope
-        """
-        path = parsed_pfn['path']
-        scope = path.lstrip('/').split('/')[0]
-        name = parsed_pfn['name']
-        return name, scope
-
-    @staticmethod
-    def _atlas(parsed_pfn: Mapping[str, str]) -> tuple[str, str]:
-        """ Translate pfn to name/scope pair
-
-        :param parsed_pfn: dictionary representing pfn containing:
-            - path: str,
-            - name: str
-        :return: tuple containing name, scope
-        """
-        path = parsed_pfn['path']
-        if path.startswith('/user') or path.startswith('/group'):
-            scope = '%s.%s' % (path.split('/')[1], path.split('/')[2])
-            name = parsed_pfn['name']
-        else:
-            name, scope = RSEDeterministicScopeTranslation._default(parsed_pfn)
-
-        return name, scope
-
-
-RSEDeterministicScopeTranslation()
-
-
-RSEDeterministicTranslationT = TypeVar('RSEDeterministicTranslationT', bound='RSEDeterministicTranslation')
-
-
-class RSEDeterministicTranslation(PolicyPackageAlgorithms):
-    """
-    Execute the logic for translating a LFN to a path.
-    """
-
-    _DEFAULT_LFN2PFN = "hash"
-    _algorithm_type = "lfn2pfn"
-
-    def __init__(self, rse=None, rse_attributes=None, protocol_attributes=None):
-        """
-        Initialize a translator object from the RSE, its attributes, and the protocol-specific
-        attributes.
-
-        :param rse: Name of RSE for this translation.
-        :param rse_attributes: A dictionary of RSE-specific attributes for use in the translation.
-        :param protocol_attributes: A dictionary of RSE/protocol-specific attributes.
-        """
-        super().__init__()
-        self.rse = rse
-        self.rse_attributes = rse_attributes if rse_attributes else {}
-        self.protocol_attributes = protocol_attributes if protocol_attributes else {}
-
-    @classmethod
-    def supports(cls, name):
-        """
-        Check to see if a specific algorithm is supported.
-
-        :param name: Name of the deterministic algorithm.
-        :returns: True if `name` is an algorithm supported by the translator class, False otherwise
-        """
-        return super()._supports(cls._algorithm_type, name)
-
-    @classmethod
-    def register(cls, lfn2pfn_callable, name=None):
-        """
-        Provided a callable function, register it as one of the valid LFN2PFN algorithms.
-
-        The callable will receive five arguments:
-         - scope: Scope of the LFN.
-         - name: LFN's path name
-         - rse: RSE name the translation is being done for.
-         - rse_attributes: Attributes of the RSE.
-         - protocol_attributes: Attributes of the RSE's protocol
-        The return value should be the last part of the PFN - it will be appended to the
-        rest of the URL.
-
-        :param lfn2pfn_callable: Callable function to use for generating paths.
-        :param name: Algorithm name used for registration.  If None, then `lfn2pfn_callable.__name__` is used.
-        """
-        if name is None:
-            name = lfn2pfn_callable.__name__
-        algorithm_dict = {name: lfn2pfn_callable}
-        super()._register(cls._algorithm_type, algorithm_dict)
-
-    @staticmethod
-    def __hash(scope, name, rse, rse_attrs, protocol_attrs):
-        """
-        Given a LFN, turn it into a sub-directory structure using a hash function.
-
-        This takes the MD5 of the LFN and uses the first four characters as a subdirectory
-        name.
-
-        :param scope: Scope of the LFN.
-        :param name: File name of the LFN.
-        :param rse: RSE for PFN (ignored)
-        :param rse_attrs: RSE attributes for PFN (ignored)
-        :param protocol_attrs: RSE protocol attributes for PFN (ignored)
-        :returns: Path for use in the PFN generation.
-        """
-        del rse
-        del rse_attrs
-        del protocol_attrs
-        hstr = hashlib.md5(('%s:%s' % (scope, name)).encode('utf-8')).hexdigest()
-        if scope.startswith('user') or scope.startswith('group'):
-            scope = scope.replace('.', '/')
-        return '%s/%s/%s/%s' % (scope, hstr[0:2], hstr[2:4], name)
-
-    @staticmethod
-    def __identity(scope, name, rse, rse_attrs, protocol_attrs):
-        """
-        Given a LFN, convert it directly to a path using the mapping:
-
-            scope:path -> scope/path
-
-        :param scope: Scope of the LFN.
-        :param name: File name of the LFN.
-        :param rse: RSE for PFN (ignored)
-        :param rse_attrs: RSE attributes for PFN (ignored)
-        :param protocol_attrs: RSE protocol attributes for PFN (ignored)
-        :returns: Path for use in the PFN generation.
-        """
-        del rse
-        del rse_attrs
-        del protocol_attrs
-        if scope.startswith('user') or scope.startswith('group'):
-            scope = scope.replace('.', '/')
-        return '%s/%s' % (scope, name)
-
-    @staticmethod
-    def __belleii(scope, name, rse, rse_attrs, protocol_attrs):
-        """
-        Given a LFN, convert it directly to a path using the mapping:
-
-            path -> path
-        This is valid only for the belleii convention where the scope can be determined
-        from the LFN using a determinitic function.
-
-        :param scope: Scope of the LFN.
-        :param name: File name of the LFN.
-        :param rse: RSE for PFN (ignored)
-        :param rse_attrs: RSE attributes for PFN (ignored)
-        :param protocol_attrs: RSE protocol attributes for PFN (ignored)
-        :returns: Path for use in the PFN generation.
-        """
-        del scope
-        del rse
-        del rse_attrs
-        del protocol_attrs
-        return name
-
-    @staticmethod
-    def __ligo(scope, name, rse, rse_attrs, protocol_attrs):
-        """
-        Given a LFN, convert it directly to a path using the Caltech schema
-
-        e.g.,: ER8:H-H1_HOFT_C02-1126256640-4096 ->
-               ER8/hoft_C02/H1/H-H1_HOFT_C02-11262/H-H1_HOFT_C02-1126256640-4096
-
-        :param scope: Scope of the LFN (observing run: ER8, O2, postO1, ...)
-        :param name: File name of the LFN (E.g., H-H1_HOFT_C02-1126256640-4096.gwf)
-        :param rse: RSE for PFN (ignored)
-        :param rse_attrs: RSE attributes for PFN (ignored)
-        :param protocol_attrs: RSE protocol attributes for PFN (ignored)
-        :returns: Path for use in the PFN generation.
-        """
-        del rse
-        del rse_attrs
-        del protocol_attrs
-        from ligo_rucio import lfn2pfn as ligo_lfn2pfn  # pylint: disable=import-error
-        return ligo_lfn2pfn.ligo_lab(scope, name, None, None, None)
-
-    @staticmethod
-    def __xenon(scope, name, rse, rse_attrs, protocol_attrs):
-        """
-        Given a LFN, turn it into a two level sub-directory structure based on the scope
-        plus a third level based on the name
-        :param scope: Scope of the LFN.
-        :param name: File name of the LFN.
-        :param rse: RSE for PFN (ignored)
-        :param rse_attrs: RSE attributes for PFN (ignored)
-        :param protocol_attrs: RSE protocol attributes for PFN (ignored)
-        :returns: Path for use in the PFN generation.
-        """
-        del rse
-        del rse_attrs
-        del protocol_attrs
-
-        return '%s/%s/%s/%s' % (scope[0:7], scope[4:len(scope)], name.split('-')[0] + "-" + name.split('-')[1], name)
-
-    @classmethod
-    def _module_init_(cls):
-        """
-        Initialize the class object on first module load.
-        """
-        cls.register(cls.__hash, "hash")
-        cls.register(cls.__identity, "identity")
-        cls.register(cls.__ligo, "ligo")
-        cls.register(cls.__belleii, "belleii")
-        cls.register(cls.__xenon, "xenon")
-        policy_module = None
-        try:
-            policy_module = config.config_get('policy', 'lfn2pfn_module')
-        except (NoOptionError, NoSectionError):
-            pass
-        if policy_module:
-            # TODO: The import of importlib is done like this due to a dependency issue with python 2.6 and incompatibility of the module with py3.x
-            # More information https://github.com/rucio/rucio/issues/875
-            import importlib
-            importlib.import_module(policy_module)
-
-        cls._DEFAULT_LFN2PFN = config.get_lfn2pfn_algorithm_default()
-
-    def path(self, scope, name):
-        """ Transforms the logical file name into a PFN's path.
-
-            :param lfn: filename
-            :param scope: scope
-
-            :returns: RSE specific URI of the physical file
-        """
-        algorithm = self.rse_attributes.get(RseAttr.LFN2PFN_ALGORITHM, 'default')
-        if algorithm == 'default':
-            algorithm = RSEDeterministicTranslation._DEFAULT_LFN2PFN
-        algorithm_callable = super()._get_one_algorithm(RSEDeterministicTranslation._algorithm_type, algorithm)
-        return algorithm_callable(scope, name, self.rse, self.rse_attributes, self.protocol_attributes)
-
-
-RSEDeterministicTranslation._module_init_()  # pylint: disable=protected-access
-
-
-class RSEProtocol:
-    """ This class is virtual and acts as a base to inherit new protocols from. It further provides some common functionality which applies for the amjority of the protocols."""
-
-    def __init__(self, protocol_attr, rse_settings, logger=logging.log):
+    def __init__(
+            self,
+            protocol_attr: dict[str, Any],
+            rse_settings: "RSESettingsDict",
+            logger: "LoggerFunction" = logging.log
+    ):
         """ Initializes the object with information about the referred RSE.
 
             :param protocol_attr:  Properties of the requested protocol.
             :param rse_settting:   The RSE settings.
             :param logger:         Optional decorated logger that can be passed from the calling daemons or servers.
         """
+
+        if 'auth_token' not in protocol_attr:
+            raise exception.NoAuthInformation('No authentication token passed for the RSE protocol')
+
         self.auth_token = protocol_attr['auth_token']
         protocol_attr.pop('auth_token')
         self.attributes = protocol_attr
@@ -356,7 +80,10 @@ class RSEProtocol:
             if getattr(rsemanager, 'SERVER_MODE', None):
                 setattr(self, '_get_path', self._get_path_nondeterministic_server)
 
-    def lfns2pfns(self, lfns):
+    def lfns2pfns(
+            self,
+            lfns: Union[list["LFNDict"], "LFNDict"]
+    ) -> dict[str, str]:
         """
             Returns a fully qualified PFN for the file referred by path.
 
@@ -398,7 +125,10 @@ class RSEProtocol:
                     self.logger(logging.WARNING, str(e))
         return pfns
 
-    def __lfns2pfns_client(self, lfns):
+    def __lfns2pfns_client(
+            self,
+            lfns: Union[list["DIDDict"], "DIDDict"]
+    ) -> dict[str, str]:
         """ Provides the path of a replica for non-deterministic sites. Will be assigned to get path by the __init__ method if necessary.
 
             :param scope: list of DIDs
@@ -411,7 +141,10 @@ class RSEProtocol:
         lfn_query = ["%s:%s" % (lfn['scope'], lfn['name']) for lfn in lfns]
         return client.lfns2pfns(self.rse['rse'], lfn_query, scheme=self.attributes['scheme'])
 
-    def _get_path(self, scope, name):
+    def _get_path(
+            self,
+            scope: str,
+            name: str):
         """ Transforms the logical file name into a PFN.
             Suitable for sites implementing the RUCIO naming convention.
             This implementation is only invoked if the RSE is deterministic.
@@ -421,26 +154,33 @@ class RSEProtocol:
 
             :returns: RSE specific URI of the physical file
         """
-        return self.translator.path(scope, name)
+        return self.translator.path(scope, name)  # type: ignore (translator could be none)
 
-    def _get_path_nondeterministic_server(self, scope, name):  # pylint: disable=invalid-name
+    def _get_path_nondeterministic_server(  # pylint: disable=invalid-name
+            self,
+            scope: str,
+            name: str
+    ) -> str:
         """ Provides the path of a replica for non-deterministic sites. Will be assigned to get path by the __init__ method if necessary. """
         vo = get_rse_vo(self.rse['id'])  # pylint: disable=E0601
-        scope = InternalScope(scope, vo=vo)  # pylint: disable=E0601
-        rep = replica.get_replica(scope=scope, name=name, rse_id=self.rse['id'])  # pylint: disable=E0601
+        internal_scope = InternalScope(scope, vo=vo)  # pylint: disable=E0601
+        rep = replica.get_replica(scope=internal_scope, name=name, rse_id=self.rse['id'])  # pylint: disable=E0601
         if 'path' in rep and rep['path'] is not None:
             path = rep['path']
         elif 'state' in rep and (rep['state'] is None or rep['state'] == 'UNAVAILABLE'):
-            raise exception.ReplicaUnAvailable('Missing path information and state is UNAVAILABLE for replica %s:%s on non-deterministic storage named %s' % (scope, name, self.rse['rse']))
+            raise exception.ReplicaUnAvailable('Missing path information and state is UNAVAILABLE for replica %s:%s on non-deterministic storage named %s' % (internal_scope, name, self.rse['rse']))
         else:
-            raise exception.ReplicaNotFound('Missing path information for replica %s:%s on non-deterministic storage named %s' % (scope, name, self.rse['rse']))
+            raise exception.ReplicaNotFound('Missing path information for replica %s:%s on non-deterministic storage named %s' % (internal_scope, name, self.rse['rse']))
         if path.startswith('/'):
             path = path[1:]
         if path.endswith('/'):
             path = path[:-1]
         return path
 
-    def parse_pfns(self, pfns):
+    def parse_pfns(
+            self,
+            pfns: Union['Iterable[str]', str]
+    ) -> dict[str, dict[str, str]]:
         """
             Splits the given PFN into the parts known by the protocol. It is also checked if the provided protocol supports the given PFNs.
 
@@ -494,7 +234,10 @@ class RSEProtocol:
 
         return ret
 
-    def exists(self, path):
+    def exists(
+            self,
+            path: Optional[str]
+    ) -> bool:
         """
             Checks if the requested file is known by the referred RSE.
 
@@ -506,7 +249,8 @@ class RSEProtocol:
         """
         raise NotImplementedError
 
-    def connect(self):
+    @abstractmethod
+    def connect(self) -> None:
         """
             Establishes the actual connection to the referred RSE.
 
@@ -514,11 +258,18 @@ class RSEProtocol:
         """
         raise NotImplementedError
 
-    def close(self):
+    @abstractmethod
+    def close(self) -> None:
         """ Closes the connection to RSE."""
         raise NotImplementedError
 
-    def get(self, path, dest, transfer_timeout=None):
+    @abstractmethod
+    def get(
+            self,
+            path: str,
+            dest: str,
+            transfer_timeout: Optional[int] = None
+    ) -> None:
         """
             Provides access to files stored inside connected the RSE.
 
@@ -532,7 +283,14 @@ class RSEProtocol:
          """
         raise NotImplementedError
 
-    def put(self, source, target, source_dir, transfer_timeout=None):
+    @abstractmethod
+    def put(
+            self,
+            source: str,
+            target: str,
+            source_dir: Optional[str],
+            transfer_timeout: Optional[int] = None
+    ) -> None:
         """
             Allows to store files inside the referred RSE.
 
@@ -547,7 +305,11 @@ class RSEProtocol:
         """
         raise NotImplementedError
 
-    def delete(self, path):
+    @abstractmethod
+    def delete(
+            self,
+            path: str
+    ) -> None:
         """
             Deletes a file from the connected RSE.
 
@@ -558,7 +320,12 @@ class RSEProtocol:
         """
         raise NotImplementedError
 
-    def rename(self, path, new_path):
+    @abstractmethod
+    def rename(
+            self,
+            path: str,
+            new_path: str
+    ) -> None:
         """ Allows to rename a file stored inside the connected RSE.
 
             :param path: path to the current file on the storage
@@ -570,17 +337,17 @@ class RSEProtocol:
         """
         raise NotImplementedError
 
-    def get_space_usage(self):
+    def get_space_usage(self) -> tuple[int, int]:
         """
             Get RSE space usage information.
 
-            :returns: a list with dict containing 'totalsize' and 'unusedsize'
+            :returns: a tuple 'totalsize' and 'unusedsize'
 
             :raises ServiceUnavailable: if some generic error occurred in the library.
         """
         raise NotImplementedError
 
-    def stat(self, path):
+    def stat(self, path: str) -> dict[str, Any]:
         """
             Returns the stats of a file.
 

rucio/rse/protocols/rclone.py

@@ -17,8 +17,9 @@ import logging
 import os
 
 from rucio.common import exception
+from rucio.common.checksum import PREFERRED_CHECKSUM
 from rucio.common.config import get_config_dirs
-from rucio.common.utils import PREFERRED_CHECKSUM, execute
+from rucio.common.utils import execute
 from rucio.rse.protocols import protocol
 
 

rucio/rse/protocols/rfio.py

@@ -37,7 +37,7 @@ class Default(protocol.RSEProtocol):
             :raises RSEAccessDenied: if no connection could be established.
         """
         extended_attributes = self.rse['protocol']['extended_attributes']
-        if 'STAGE_SVCCLASS' in extended_attributes:
+        if isinstance(extended_attributes, dict) and 'STAGE_SVCCLASS' in extended_attributes:
             os.environ['STAGE_SVCCLASS'] = extended_attributes['STAGE_SVCCLASS']
 
     def path2pfn(self, path):
@@ -134,3 +134,12 @@ class Default(protocol.RSEProtocol):
         ret['path'] = ret['path'].partition(ret['name'])[0]
 
         return ret
+
+    def delete(self, path):
+        raise NotImplementedError
+
+    def get(self, path, dest, transfer_timeout=None):
+        raise NotImplementedError
+
+    def rename(self, path, new_path):
+        raise NotImplementedError

rucio/rse/protocols/ssh.py

@@ -17,7 +17,8 @@ import os
 import re
 
 from rucio.common import exception
-from rucio.common.utils import PREFERRED_CHECKSUM, execute
+from rucio.common.checksum import PREFERRED_CHECKSUM
+from rucio.common.utils import execute
 from rucio.rse.protocols import protocol
 
 

rucio/rse/protocols/storm.py

@@ -68,17 +68,6 @@ class Default(protocol.RSEProtocol):
         """
         return ''.join([self.rse['scheme'], '://%s' % self.rse['hostname'], path])
 
-    def exists(self, pfn):
-        """ Checks if the requested file is known by the referred RSE.
-
-            :param pfn: Physical file name
-
-            :returns: True if the file exists, False if it doesn't
-
-            :raise  ServiceUnavailable
-        """
-        raise NotImplementedError
-
     def connect(self):
         """ Establishes the actual connection to the referred RSE.
 
@@ -139,7 +128,7 @@ class Default(protocol.RSEProtocol):
             name = pfn.split('/')[-1]
             if name not in target:
                 target = None
-        except:
+        except Exception:
             target = None
             pass
 
@@ -149,7 +138,7 @@ class Default(protocol.RSEProtocol):
             # requests preferable
             try:
                 rcode, etag_meta = requests_etag(pfn, 300)
-            except:
+            except Exception:
                 pass
             # fallback to davix
             if rcode != 207: