rucio 35.7.0__py3-none-any.whl → 37.0.0rc2__py3-none-any.whl

rucio/common/config.py

@@ -17,18 +17,23 @@
 import configparser
 import json
 import os
-from collections.abc import Callable
+from functools import cache
 from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union, overload
 
 from rucio.common import exception
-from rucio.common.exception import ConfigNotFound, DatabaseException
+from rucio.common.exception import ConfigLoadingError, ConfigNotFound, DatabaseException
 
 _T = TypeVar('_T')
 _U = TypeVar('_U')
 
 if TYPE_CHECKING:
+    from collections.abc import Callable
+
     from sqlalchemy.orm import Session
 
+LEGACY_SECTION_NAME = {}
+LEGACY_OPTION_NAME = {}
+
 
 def convert_to_any_type(value: str) -> Union[bool, int, float, str]:
     if value.lower() in ['true', 'yes', 'on']:
@@ -110,7 +115,7 @@ def config_get(
         session: "Optional[Session]" = ...,
         use_cache: bool = ...,
         expiration_time: int = ...,
-        convert_type_fnc: Callable[[str], _T],
+        convert_type_fnc: 'Callable[[str], _T]',
 ) -> _T:
     ...
 
@@ -126,7 +131,7 @@ def config_get(
         session: "Optional[Session]" = ...,
         use_cache: bool = ...,
         expiration_time: int = ...,
-        convert_type_fnc: Callable[[str], _U],
+        convert_type_fnc: 'Callable[[str], _U]',
 ) -> Union[_T, _U]:
     ...
 
@@ -143,7 +148,7 @@ def config_get(
         session: "Optional[Session]" = ...,
         use_cache: bool = ...,
         expiration_time: int = ...,
-        convert_type_fnc: Callable[[str], _U],
+        convert_type_fnc: 'Callable[[str], _U]',
 ) -> Union[_T, _U]:
     ...
 
@@ -158,7 +163,7 @@ def config_get(
         session: "Optional[Session]" = None,
         use_cache: bool = True,
         expiration_time: int = 900,
-        convert_type_fnc: Callable[[str], _T] = lambda x: x,
+        convert_type_fnc: 'Callable[[str], _T]' = lambda x: x,
 ) -> Union[_T, _U]:
     """
     Return the string value for a given option in a section
@@ -196,7 +201,7 @@ def config_get(
         except ConfigNotFound:
             pass
 
-        from rucio.common.utils import is_client
+        from rucio.common.client import is_client
         client_mode = is_client()
 
         if not client_mode and check_config_table:
@@ -223,8 +228,6 @@ def get_legacy_config(section: str, option: str):
     :param option: The option of the new config.
     :returns: The string value of the legacy option if one is found, None otherwise.
     """
-    LEGACY_SECTION_NAME = {}
-    LEGACY_OPTION_NAME = {}
 
     section = LEGACY_SECTION_NAME.get(section, section)
     option = LEGACY_OPTION_NAME.get(option, option)
@@ -544,12 +547,12 @@ def config_get_list(
         section: str,
         option: str,
         *,
-        default: list[str] = ...,
+        default: _T = ...,
         check_config_table: bool = ...,
         session: "Optional[Session]" = ...,
         use_cache: bool = ...,
         expiration_time: int = ...,
-) -> list[str]:
+) -> Union[list[str], _T]:
     ...
 
 
@@ -637,7 +640,7 @@ def __config_get_table(
         session: "Optional[Session]" = None,
         use_cache: bool = True,
         expiration_time: int = 900,
-        convert_type_fnc: Optional[Callable[[str], _T]],
+        convert_type_fnc: Optional['Callable[[str], _T]'],
 ) -> _T:
     """
     Search for a section-option configuration parameter in the configuration table
@@ -655,7 +658,6 @@ def __config_get_table(
     :raises ConfigNotFound
     :raises DatabaseException
     """
-    global __CONFIG
     try:
         from rucio.core.config import get as core_config_get
         return core_config_get(section, option, default=default, session=session, use_cache=use_cache,
@@ -664,7 +666,7 @@ def __config_get_table(
         if raise_exception and default is None:
             raise err
         if clean_cached:
-            __CONFIG = None
+            clean_cached_config()
         return default
 
 
@@ -714,14 +716,8 @@ def get_config_dirs() -> list[str]:
     """
     configdirs = []
 
-    if 'RUCIO_HOME' in os.environ:
-        configdirs.append('%s/etc/' % os.environ['RUCIO_HOME'])
-
-    if 'VIRTUAL_ENV' in os.environ:
-        configdirs.append('%s/etc/' % os.environ['VIRTUAL_ENV'])
-
-    if 'CONDA_PREFIX' in os.environ:
-        configdirs.append('%s/etc/' % os.environ['CONDA_PREFIX'])
+    env_vars = ['RUCIO_HOME', 'VIRTUAL_ENV', 'CONDA_PREFIX']
+    configdirs.extend([os.path.join(os.environ[var], 'etc', '') for var in env_vars if var in os.environ])
 
     configdirs.append('/opt/rucio/etc/')
 
@@ -758,21 +754,15 @@ def get_rse_credentials(path_to_credentials_file: Optional[Union[str, os.PathLik
     return credentials
 
 
-__CONFIG = None
-
-
+@cache
 def get_config() -> configparser.ConfigParser:
     """Factory function for the configuration class. Returns the ConfigParser instance."""
-    global __CONFIG
-    if __CONFIG is None:
-        __CONFIG = Config()
-    return __CONFIG.parser
+    return Config().parser
 
 
 def clean_cached_config() -> None:
     """Deletes the cached config singleton instance."""
-    global __CONFIG
-    __CONFIG = None
+    get_config.cache_clear()
 
 
 class Config:
@@ -795,7 +785,4 @@ class Config:
                     '\n\t' + '\n\t'.join(configs))
 
         if not self.parser.read(self.configfile) == [self.configfile]:
-            raise ConfigNotFound(
-                'Could not load Rucio configuration file. '
-                'Rucio tried loading the following configuration file:'
-                '\n\t' + self.configfile)
+            raise ConfigLoadingError(self.configfile)

rucio/common/constants.py

@@ -48,7 +48,7 @@ if config_get_bool('transfers', 'srm_https_compatibility', raise_exception=False
     SCHEME_MAP['srm'].append('davs')
     SCHEME_MAP['davs'].append('srm')
 
-SORTING_ALGORITHMS_LITERAL = Literal['geoip', 'closeness', 'custom_table', 'dynamic', 'ranking', 'random']
+SORTING_ALGORITHMS_LITERAL = Literal['geoip', 'custom_table', 'random']
 SORTING_ALGORITHMS = list(get_args(SORTING_ALGORITHMS_LITERAL))
 
 SUPPORTED_PROTOCOLS_LITERAL = Literal['gsiftp', 'srm', 'root', 'davs', 'http', 'https', 'file', 'storm', 'srm+https', 'scp', 'rsync', 'rclone', 'magnet']
@@ -56,8 +56,11 @@ SUPPORTED_PROTOCOLS: list[str] = list(get_args(SUPPORTED_PROTOCOLS_LITERAL))
 
 RSE_SUPPORTED_PROTOCOL_DOMAINS_LITERAL = Literal['ALL', 'LAN', 'WAN']
 
-RSE_SUPPORTED_PROTOCOL_OPERATIONS_LITERAL = Literal['read', 'write', 'delete', 'third_party_copy_read', 'third_party_copy_write']
-RSE_SUPPORTED_PROTOCOL_OPERATIONS: list[str] = list(get_args(RSE_SUPPORTED_PROTOCOL_OPERATIONS_LITERAL))
+RSE_BASE_SUPPORTED_PROTOCOL_OPERATIONS_LITERAL = Literal['read', 'write', 'delete']
+RSE_BASE_SUPPORTED_PROTOCOL_OPERATIONS: list[str] = list(get_args(RSE_BASE_SUPPORTED_PROTOCOL_OPERATIONS_LITERAL))
+
+RSE_ALL_SUPPORTED_PROTOCOL_OPERATIONS_LITERAL = Literal[RSE_BASE_SUPPORTED_PROTOCOL_OPERATIONS_LITERAL, 'third_party_copy_read', 'third_party_copy_write']
+RSE_ALL_SUPPORTED_PROTOCOL_OPERATIONS: list[str] = list(get_args(RSE_ALL_SUPPORTED_PROTOCOL_OPERATIONS_LITERAL))
 
 FTS_STATE = namedtuple('FTS_STATE', ['SUBMITTED', 'READY', 'ACTIVE', 'FAILED', 'FINISHED', 'FINISHEDDIRTY', 'NOT_USED',
                                      'CANCELED'])('SUBMITTED', 'READY', 'ACTIVE', 'FAILED', 'FINISHED', 'FINISHEDDIRTY',
@@ -74,12 +77,14 @@ FTS_JOB_TYPE = namedtuple('FTS_JOB_TYPE', ['MULTIPLE_REPLICA', 'MULTI_HOP', 'SES
 MAX_MESSAGE_LENGTH = 4000
 
 
+@enum.unique
 class SuspiciousAvailability(enum.Enum):
     ALL = 0
     EXIST_COPIES = 1
     LAST_COPY = 2
 
 
+@enum.unique
 class ReplicaState(enum.Enum):
     # From rucio.db.sqla.constants, update that file at the same time as this
     AVAILABLE = 'A'
@@ -157,3 +162,56 @@ class RseAttr:
     DEFAULT_LIMIT_FILES = 'default_limit_files'
     QUOTA_APPROVERS = 'quota_approvers'
     RULE_DELETERS = 'rule_deleters'
+
+
+# Literal types to allow overloading of functions with RSE attributes in their signature.
+# RSE attributes are encoded via the BooleanString decorator as VARCHAR in the database,
+# but they are used as either bool or string in the code.
+# This is only determined at runtime, so for static type checking
+# we need to manually specify which attrs are string and which are bool.
+# In future, we could refactor RseAttr to avoid code duplication.
+RSE_ATTRS_STR = Literal[
+    'archive_timeout',
+    'associated_sites',
+    'bittorrent_tracker_addr',
+    'country',
+    'decommission',
+    'default_account_limit_bytes',
+    'fts',
+    'globus_endpoint_id',
+    'lfn2pfn_algorithm',
+    'maximum_pin_lifetime',
+    'multihop_tombstone_delay',
+    'naming_convention',
+    'oidc_base_path',
+    'oidc_support'
+    'physgroup',
+    'qbittorrent_management_address'
+    'rule_approvers',
+    's3_url_style',
+    'simulate_multirange',
+    'site',
+    'source_for_total_space',
+    'source_for_used_space',
+    'staging_buffer',
+    'tombstone_delay',
+    'type'
+]
+
+RSE_ATTRS_BOOL = Literal[
+    'auto_approve_bytes',
+    'auto_approve_files',
+    'block_manual_approval',
+    'greedyDeletion',
+    'is_object_store',
+    'restricted_read',
+    'restricted_write',
+    'skip_upload_stat',
+    'staging_required',
+    'strict_copy',
+    'use_ipv4',
+    'verify_checksum'
+]
+
+SUPPORTED_SIGN_URL_SERVICES_LITERAL = Literal['gcs', 's3', 'swift']
+SUPPORTED_SIGN_URL_SERVICES = list(get_args(SUPPORTED_SIGN_URL_SERVICES_LITERAL))

rucio/common/didtype.py

@@ -16,7 +16,7 @@
 DID type to represent a did and to simplify operations on it
 """
 
-from typing import Union
+from typing import Any, Union
 
 from rucio.common.exception import DIDError
 
@@ -58,6 +58,22 @@ class DID:
         self.scope: str = ''
         self.name: str = ''
 
+        did = self._parse_did_from_args(*args, **kwargs)
+
+        self._construct_did(did)
+
+        if not self.is_valid_format():
+            raise DIDError('Object has invalid format after construction: {}'.format(str(self)))
+
+    def _parse_did_from_args(
+            self,
+            *args,
+            **kwargs
+    ) -> Union["DID", str, tuple[str, str], list[str], dict[str, str]]:
+        """
+        Parse the DID object from the given arguments
+        :return: DID object
+        """
         num_args = len(args)
         num_kwargs = len(kwargs)
         if (num_args + num_kwargs) > 2:
@@ -77,46 +93,78 @@ class DID:
                     else:
                         raise DIDError('Constructor got unexpected keyword argument: {}'.format(k))
                 else:
-                    raise DIDError('First argument of constructor is expected to be string type'
+                    raise DIDError('First argument of constructor is expected to be string type '
                                    'when keyword argument is given. Given type: {}'.format(type(did)))
         elif num_args == 0:
             did = kwargs.get('did', kwargs)
         else:
             did = args
+        return did
 
+    def _construct_did(self, did: Any) -> None:
+        """
+        Construct the DID object from the given input.
+
+        :param did: input to construct the DID object from
+        """
         if isinstance(did, dict):
-            self.scope = did.get('scope', '')
-            self.name = did.get('name', '')
-            if not self.has_scope():
-                self.update_implicit_scope()
+            self._did_from_dict(did)
         elif isinstance(did, tuple) or isinstance(did, list):
-            if len(did) != 2:
-                raise DIDError('Construction from tuple or list requires exactly 2 elements')
-            self.scope = did[0]
-            self.name = did[1]
+            self._did_from_list_or_tuple(did)
         elif isinstance(did, str):
-            did_parts = did.split(DID.SCOPE_SEPARATOR, 1)
-            if len(did_parts) == 1:
-                self.name = did
-                self.update_implicit_scope()
-                if not self.has_scope():
-                    raise DIDError('Object construction from non-splitable string is ambigious')
-            else:
-                self.scope = did_parts[0]
-                self.name = did_parts[1]
+            self._did_from_str(did)
         elif isinstance(did, DID):
-            self.scope = did.scope
-            self.name = did.name
+            self._did_from_did_object(did)
         else:
             raise DIDError('Cannot build object from: {}'.format(type(did)))
 
         if self.name.endswith('/'):
             self.name = self.name[:-1]
 
-        if not self.is_valid_format():
-            raise DIDError('Object has invalid format after construction: {}'.format(str(self)))
+    def _did_from_str(self, did: str) -> None:
+        """
+        Construct the DID from a string.
+        :param did: string containing the DID information
+        """
+        did_parts = did.split(DID.SCOPE_SEPARATOR, 1)
+        if len(did_parts) == 1:
+            self.name = did
+            self._update_implicit_scope()
+            if not self.has_scope():
+                raise DIDError('Object construction from non-splitable string is ambigious')
+        else:
+            self.scope = did_parts[0]
+            self.name = did_parts[1]
+
+    def _did_from_dict(self, did: dict[str, str]) -> None:
+        """
+        Construct the DID from a dictionary.
+        :param did: dictionary optionally containing the keys 'scope' and 'name'
+        """
+        self.scope = did.get('scope', '')
+        self.name = did.get('name', '')
+        if not self.has_scope():
+            self._update_implicit_scope()
+
+    def _did_from_list_or_tuple(self, did: Union[list[str], tuple[str, str]]) -> None:
+        """
+        Construct the DID from a list or tuple.
+        :param did: list or tuple with expected length of 2
+        """
+        if len(did) != 2:
+            raise DIDError('Construction from tuple or list requires exactly 2 elements. Number of elements passed: %i' % len(did))
+        self.scope = did[0]
+        self.name = did[1]
+
+    def _did_from_did_object(self, did: "DID") -> None:
+        """
+        Construct the DID from another DID object.
+        :param did: DID object
+        """
+        self.scope = did.scope
+        self.name = did.name
 
-    def update_implicit_scope(self) -> None:
+    def _update_implicit_scope(self) -> None:
         """
         This method sets the scope if it is implicitly given in self.name
         """

rucio/common/dumper/__init__.py

@@ -21,14 +21,21 @@ import os
 import re
 import sys
 import tempfile
+from configparser import NoOptionError, NoSectionError
+from functools import cache
 from typing import TYPE_CHECKING, Any, Literal, Optional, Union
 
-import gfal2
-import magic
+try:
+    # requires non-Python package pre-requisites that break unit tests
+    import gfal2
+except ModuleNotFoundError:
+    pass
+
 import requests
+from magic import Magic
 
 from rucio.common import config
-from rucio.core.rse import get_rse_id, get_rse_protocols
+from rucio.common.exception import ConfigNotFound
 
 if TYPE_CHECKING:
     from collections.abc import Iterator
@@ -70,7 +77,7 @@ def error(text: str, exit_code: int = 1) -> None:
     logger = logging.getLogger('dumper.__init__')
     logger.error(text)
     sys.stderr.write(text + '\n')
-    exit(1)
+    exit(exit_code)
 
 
 def mkdir(dir_: "StrOrBytesPath") -> None:
@@ -89,14 +96,21 @@ def cacert_config(config: "ModuleType", rucio_home: str) -> Optional[Union["File
     logger = logging.getLogger('dumper.__init__')
     try:
         cacert = config.config_get('client', 'ca_cert').replace('$RUCIO_HOME', rucio_home)
-    except KeyError:
+    except ConfigNotFound:
+        logger.warning('Configuration file not found.')
+        cacert = None
+    except (KeyError, NoOptionError, NoSectionError):
+        logger.warning('CA Certificate configuration not found.')
         cacert = None
 
-    if cacert is None or not os.path.exists(cacert):
-        logger.warning('Configured CA Certificate file "%s" not found: Host certificate verification disabled', cacert)
-        cacert = False
+    if cacert and os.path.exists(cacert):
+        return cacert
+    else:
+        logger.warning('Configured CA Certificate file "%s" not found', cacert)
+
+    logger.warning('Host certificate verification disabled.')
 
-    return cacert
+    return False
 
 
 def rucio_home() -> str:
@@ -115,41 +129,37 @@ RESULTS_DIR = 'results'
 CHUNK_SIZE = 4194304  # 4MiB
 
 
-# There are two Python modules with the name `magic`, luckily both do
-# the same thing.
-# pylint: disable=no-member
-if 'open' in dir(magic):
-    _mime = magic.open(magic.MAGIC_MIME)
-    _mime.load()
-    mimetype = _mime.file
-else:
-    mimetype = lambda filename: magic.from_file(filename, mime=True)  # NOQA
-# pylint: enable=no-member
+@cache
+def get_libmagic_wrapper() -> Magic:
+    """
+    Returns a libmagic wrapper.
+    """
+    return Magic(mime=True)
 
 
-def isplaintext(filename: "GenericPath") -> bool:
-    '''
+def is_plaintext(filename: "GenericPath") -> bool:
+    """
     Returns True if `filename` has mimetype == 'text/plain'.
-    '''
-    if os.path.islink(filename):
-        filename = os.readlink(filename)
-    return mimetype(filename).split(';')[0] == 'text/plain'
+    """
+    mime = get_libmagic_wrapper()
+    return mime.from_file(filename) == 'text/plain'
 
 
-def smart_open(filename: "GenericPath") -> Optional[Union["TextIO", gzip.GzipFile]]:
+def smart_open(filename: "GenericPath") -> Optional["TextIO"]:
     '''
     Returns an open file object if `filename` is plain text, else assumes
     it is a bzip2 compressed file and returns a file-like object to
     handle it.
     '''
     f = None
-    if isplaintext(filename):
+    if is_plaintext(filename):
         f = open(filename, 'rt')
     else:
-        file_type = mimetype(filename)
-        if file_type.find('gzip') > -1:
-            f = gzip.GzipFile(filename, 'rt')
-        elif file_type.find('bzip2') > -1:
+        mime = get_libmagic_wrapper()
+        file_type = mime.from_file(filename)
+        if file_type in ['application/gzip', 'application/x-gzip']:
+            f = gzip.open(filename, 'rt')
+        elif file_type == 'application/x-bzip2':
             f = bz2.open(filename, 'rt')
         else:
             pass  # Not supported format
@@ -204,8 +214,6 @@ def temp_file(
 
 
 DATETIME_FORMAT = '%Y-%m-%d %H:%M:%S'
-DATETIME_FORMAT_FULL = '%Y-%m-%dT%H:%M:%S'
-MILLISECONDS_RE = re.compile(r'\.(\d{3})Z$')
 
 
 def to_datetime(str_or_datetime: Union[datetime.datetime, str]) -> Optional[datetime.datetime]:
@@ -236,17 +244,16 @@ def to_datetime(str_or_datetime: Union[datetime.datetime, str]) -> Optional[date
             'Trying to parse "%s" date with resolution of milliseconds',
             str_or_datetime,
         )
-        milliseconds = int(MILLISECONDS_RE.search(str_or_datetime).group(1))
-        str_or_datetime = MILLISECONDS_RE.sub('', str_or_datetime)
         date = datetime.datetime.strptime(
             str_or_datetime,
-            DATETIME_FORMAT,
-        )
-        date = date + datetime.timedelta(microseconds=milliseconds * 1000)
+            # Adding milliseconds to the datetime format
+            DATETIME_FORMAT + '.%f',
+        ).replace(microsecond=0)
     return date
 
 
 def ddmendpoint_preferred_protocol(ddmendpoint: str) -> "RSEProtocolDict":
+    from rucio.core.rse import get_rse_id, get_rse_protocols  # pylint: disable=import-outside-toplevel
     return next(p for p in get_rse_protocols(get_rse_id(ddmendpoint))['protocols'] if p['domains']['wan']['read'] == 1)