rucio 36.1.0__tar.gz → 36.2.0__tar.gz

{rucio-36.1.0 → rucio-36.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rucio
-Version: 36.1.0
+Version: 36.2.0
 Summary: Rucio Package
 Home-page: https://rucio.cern.ch/
 Author: Rucio

{rucio-36.1.0 → rucio-36.2.0}/bin/rucio

@@ -21,6 +21,16 @@ from rucio.client.commands.bin_legacy.rucio import main as main_legacy
 from rucio.client.commands.command import main
 
 
+def _get_first_command(args: list[str]) -> Optional[str]:
+    return next(
+        (
+            arg for arg in args
+            if arg != 'rucio' and (arg[0] != "-" or arg in ("-h", "--help", "--version"))
+        ),
+        None
+    )
+
+
 def make_warning():
     base_warning = "\nWARNING: This method is being deprecated."
     new_command = map_legacy_command()
@@ -33,10 +43,7 @@ def make_warning():
 
 
 def map_legacy_command() -> Optional[list[str]]:
-    try:
-        command = [arg for arg in sys.argv if arg[0] != "-" or arg in ("-h", "--version")][1]
-    except IndexError:
-        command = None
+    command = _get_first_command(sys.argv[1:])
 
     new_command = None
     if command not in ("-h", "--version", None):
@@ -89,7 +96,9 @@ def map_legacy_command() -> Optional[list[str]]:
 
 if __name__ == "__main__":
     commands = ("-h", "--help", "--version", "account", "config", "did", "replica", "rse", "rule", "scope", "subscription", "ping", "whoami", "test-server", "lifetime-exception", "upload", "download")
-    first_command = [arg for arg in sys.argv if arg[0] != "-" or arg in ("-h", "--help", "--version")][1]
+
+    first_command = _get_first_command(sys.argv[1:])
+
     is_legacy = '--legacy' in sys.argv
 
     if (first_command in commands) and not is_legacy:

{rucio-36.1.0 → rucio-36.2.0}/bin/rucio-admin

@@ -20,7 +20,10 @@ from rucio.client.commands.bin_legacy.rucio_admin import main as main_legacy
 def make_warning():
     base_warning = "\nWARNING: This method is being deprecated."
     args = [arg for arg in sys.argv if arg[0] != "-" or arg in ("-h", "--help", "--version")]
-    first_command = args[1]
+    try:
+        first_command = args[1]
+    except IndexError:
+        first_command = None
     try:
         second_command = args[2]
     except IndexError:

{rucio-36.1.0 → rucio-36.2.0}/lib/rucio/client/commands/bin_legacy/rucio_admin.py

@@ -418,7 +418,6 @@ def info_rse(args, client, logger, console, spinner):
     if cli_config == 'rich':
         table = generate_table(table_data, row_styles=['none'], col_alignments=['left', 'left'])
         output.append(table)
-        table_data = []
     else:
         print('Protocols:')
         print('==========')
@@ -430,6 +429,7 @@ def info_rse(args, client, logger, console, spinner):
         else:
             print('  ' + protocol['scheme'])
 
+        table_data = []
         for item in sorted(protocol):
             if cli_config == 'rich':
                 if item == 'domains':
@@ -447,10 +447,11 @@ def info_rse(args, client, logger, console, spinner):
                 else:
                     print('    ' + item + ': ' + str(protocol[item]))
 
-    if cli_config == 'rich':
-        table = generate_table(table_data, col_alignments=['left', 'left'], row_styles=['none'])
-        output.append(Padding.indent(table, 2))
+        if cli_config == 'rich':
+            table = generate_table(table_data, col_alignments=['left', 'left'], row_styles=['none'])
+            output.append(Padding.indent(table, 2))
 
+    if cli_config == 'rich':
         header = ['SOURCE', 'USED', 'FILES', 'FREE', 'TOTAL', 'UPDATED AT']
         key2id = {header[i].lower().replace(' ', '_'): i for i in range(len(header))}
         table_data = []

{rucio-36.1.0 → rucio-36.2.0}/lib/rucio/common/config.py

@@ -17,10 +17,11 @@
 import configparser
 import json
 import os
+from functools import cache
 from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union, overload
 
 from rucio.common import exception
-from rucio.common.exception import ConfigNotFound, DatabaseException
+from rucio.common.exception import ConfigLoadingError, ConfigNotFound, DatabaseException
 
 _T = TypeVar('_T')
 _U = TypeVar('_U')
@@ -656,7 +657,6 @@ def __config_get_table(
     :raises ConfigNotFound
     :raises DatabaseException
     """
-    global __CONFIG
     try:
         from rucio.core.config import get as core_config_get
         return core_config_get(section, option, default=default, session=session, use_cache=use_cache,
@@ -665,7 +665,7 @@ def __config_get_table(
         if raise_exception and default is None:
             raise err
         if clean_cached:
-            __CONFIG = None
+            clean_cached_config()
         return default
 
 
@@ -753,21 +753,15 @@ def get_rse_credentials(path_to_credentials_file: Optional[Union[str, os.PathLik
     return credentials
 
 
-__CONFIG = None
-
-
+@cache
 def get_config() -> configparser.ConfigParser:
     """Factory function for the configuration class. Returns the ConfigParser instance."""
-    global __CONFIG
-    if __CONFIG is None:
-        __CONFIG = Config()
-    return __CONFIG.parser
+    return Config().parser
 
 
 def clean_cached_config() -> None:
     """Deletes the cached config singleton instance."""
-    global __CONFIG
-    __CONFIG = None
+    get_config.cache_clear()
 
 
 class Config:
@@ -790,7 +784,4 @@ class Config:
                     '\n\t' + '\n\t'.join(configs))
 
         if not self.parser.read(self.configfile) == [self.configfile]:
-            raise ConfigNotFound(
-                'Could not load Rucio configuration file. '
-                'Rucio tried loading the following configuration file:'
-                '\n\t' + self.configfile)
+            raise ConfigLoadingError(self.configfile)

{rucio-36.1.0 → rucio-36.2.0}/lib/rucio/common/dumper/__init__.py

@@ -21,14 +21,21 @@ import os
 import re
 import sys
 import tempfile
+from configparser import NoOptionError, NoSectionError
+from functools import cache
 from typing import TYPE_CHECKING, Any, Literal, Optional, Union
 
-import gfal2
-import magic
+try:
+    # requires non-Python package pre-requisites that break unit tests
+    import gfal2
+except ModuleNotFoundError:
+    pass
+
 import requests
+from magic import Magic
 
 from rucio.common import config
-from rucio.core.rse import get_rse_id, get_rse_protocols
+from rucio.common.exception import ConfigNotFound
 
 if TYPE_CHECKING:
     from collections.abc import Iterator
@@ -70,7 +77,7 @@ def error(text: str, exit_code: int = 1) -> None:
     logger = logging.getLogger('dumper.__init__')
     logger.error(text)
     sys.stderr.write(text + '\n')
-    exit(1)
+    exit(exit_code)
 
 
 def mkdir(dir_: "StrOrBytesPath") -> None:
@@ -89,14 +96,21 @@ def cacert_config(config: "ModuleType", rucio_home: str) -> Optional[Union["File
     logger = logging.getLogger('dumper.__init__')
     try:
         cacert = config.config_get('client', 'ca_cert').replace('$RUCIO_HOME', rucio_home)
-    except KeyError:
+    except ConfigNotFound:
+        logger.warning('Configuration file not found.')
+        cacert = None
+    except (KeyError, NoOptionError, NoSectionError):
+        logger.warning('CA Certificate configuration not found.')
         cacert = None
 
-    if cacert is None or not os.path.exists(cacert):
-        logger.warning('Configured CA Certificate file "%s" not found: Host certificate verification disabled', cacert)
-        cacert = False
+    if cacert and os.path.exists(cacert):
+        return cacert
+    else:
+        logger.warning('Configured CA Certificate file "%s" not found', cacert)
+
+    logger.warning('Host certificate verification disabled.')
 
-    return cacert
+    return False
 
 
 def rucio_home() -> str:
@@ -115,41 +129,37 @@ RESULTS_DIR = 'results'
 CHUNK_SIZE = 4194304  # 4MiB
 
 
-# There are two Python modules with the name `magic`, luckily both do
-# the same thing.
-# pylint: disable=no-member
-if 'open' in dir(magic):
-    _mime = magic.open(magic.MAGIC_MIME)
-    _mime.load()
-    mimetype = _mime.file
-else:
-    mimetype = lambda filename: magic.from_file(filename, mime=True)  # NOQA
-# pylint: enable=no-member
+@cache
+def get_libmagic_wrapper() -> Magic:
+    """
+    Returns a libmagic wrapper.
+    """
+    return Magic(mime=True)
 
 
-def isplaintext(filename: "GenericPath") -> bool:
-    '''
+def is_plaintext(filename: "GenericPath") -> bool:
+    """
     Returns True if `filename` has mimetype == 'text/plain'.
-    '''
-    if os.path.islink(filename):
-        filename = os.readlink(filename)
-    return mimetype(filename).split(';')[0] == 'text/plain'
+    """
+    mime = get_libmagic_wrapper()
+    return mime.from_file(filename) == 'text/plain'
 
 
-def smart_open(filename: "GenericPath") -> Optional[Union["TextIO", gzip.GzipFile]]:
+def smart_open(filename: "GenericPath") -> Optional["TextIO"]:
     '''
     Returns an open file object if `filename` is plain text, else assumes
     it is a bzip2 compressed file and returns a file-like object to
     handle it.
     '''
     f = None
-    if isplaintext(filename):
+    if is_plaintext(filename):
         f = open(filename, 'rt')
     else:
-        file_type = mimetype(filename)
-        if file_type.find('gzip') > -1:
-            f = gzip.GzipFile(filename, 'rt')
-        elif file_type.find('bzip2') > -1:
+        mime = get_libmagic_wrapper()
+        file_type = mime.from_file(filename)
+        if file_type in ['application/gzip', 'application/x-gzip']:
+            f = gzip.open(filename, 'rt')
+        elif file_type == 'application/x-bzip2':
             f = bz2.open(filename, 'rt')
         else:
             pass  # Not supported format
@@ -204,8 +214,6 @@ def temp_file(
 
 
 DATETIME_FORMAT = '%Y-%m-%d %H:%M:%S'
-DATETIME_FORMAT_FULL = '%Y-%m-%dT%H:%M:%S'
-MILLISECONDS_RE = re.compile(r'\.(\d{3})Z$')
 
 
 def to_datetime(str_or_datetime: Union[datetime.datetime, str]) -> Optional[datetime.datetime]:
@@ -236,17 +244,16 @@ def to_datetime(str_or_datetime: Union[datetime.datetime, str]) -> Optional[date
             'Trying to parse "%s" date with resolution of milliseconds',
             str_or_datetime,
         )
-        milliseconds = int(MILLISECONDS_RE.search(str_or_datetime).group(1))
-        str_or_datetime = MILLISECONDS_RE.sub('', str_or_datetime)
         date = datetime.datetime.strptime(
             str_or_datetime,
-            DATETIME_FORMAT,
-        )
-        date = date + datetime.timedelta(microseconds=milliseconds * 1000)
+            # Adding milliseconds to the datetime format
+            DATETIME_FORMAT + '.%f',
+        ).replace(microsecond=0)
     return date
 
 
 def ddmendpoint_preferred_protocol(ddmendpoint: str) -> "RSEProtocolDict":
+    from rucio.core.rse import get_rse_id, get_rse_protocols  # pylint: disable=import-outside-toplevel
     return next(p for p in get_rse_protocols(get_rse_id(ddmendpoint))['protocols'] if p['domains']['wan']['read'] == 1)
 
 

{rucio-36.1.0 → rucio-36.2.0}/lib/rucio/common/dumper/consistency.py

@@ -18,10 +18,17 @@ import os
 import re
 import subprocess
 import tempfile
+from typing import TYPE_CHECKING, Any, Optional, Union, cast
 
 from rucio.common import dumper
 from rucio.common.dumper import DUMPS_CACHE_DIR, data_models, error, path_parsing
 
+if TYPE_CHECKING:
+    from argparse import Namespace, _SubParsersAction
+    from collections.abc import Callable, Iterable, Iterator
+
+    from _typeshed import SupportsNext
+
 subcommands = ['consistency', 'consistency-manual']
 
 
@@ -32,11 +39,25 @@ class Consistency(data_models.DataModel):
     )
 
     @classmethod
-    def dump(cls, subcommand, ddm_endpoint, storage_dump, prev_date_fname=None, next_date_fname=None,
-             prev_date=None, next_date=None, sort_rucio_replica_dumps=True, date=None,
-             cache_dir=DUMPS_CACHE_DIR):
+    def dump(
+        cls,
+        subcommand: str,
+        ddm_endpoint: str,
+        storage_dump: str,
+        prev_date_fname: Optional[str] = None,
+        next_date_fname: Optional[str] = None,
+        prev_date: Optional[Union[str, datetime.datetime]] = None,
+        next_date: Optional[Union[str, datetime.datetime]] = None,
+        sort_rucio_replica_dumps: bool = True,
+        date: Optional[datetime.datetime] = None,
+        cache_dir: str = DUMPS_CACHE_DIR
+    ):
         logger = logging.getLogger('auditor.consistency')
         if subcommand == 'consistency':
+            if prev_date is None:
+                prev_date = 'latest'
+            if next_date is None:
+                next_date = 'latest'
             prev_date_fname = data_models.Replica.download(
                 ddm_endpoint, prev_date, cache_dir=cache_dir)
             next_date_fname = data_models.Replica.download(
@@ -50,7 +71,7 @@ class Consistency(data_models.DataModel):
 
         prefix_components = path_parsing.components(dumper.ddmendpoint_url(ddm_endpoint))
 
-        def parser(line):
+        def parser(line: str) -> str:
             '''
             Simple parser for Rucio replica dumps.
 
@@ -63,7 +84,7 @@ class Consistency(data_models.DataModel):
 
             return ','.join((path, status))
 
-        def strip_storage_dump(line):
+        def strip_storage_dump(line: str) -> str:
             '''
             Parser to have consistent paths in storage dumps.
 
@@ -80,26 +101,26 @@ class Consistency(data_models.DataModel):
 
         if sort_rucio_replica_dumps:
             prev_date_fname_sorted = gnu_sort(
-                parse_and_filter_file(prev_date_fname, parser=parser, cache_dir=cache_dir),
+                parse_and_filter_file(prev_date_fname, parser=parser, cache_dir=cache_dir),  # type: ignore
                 delimiter=',',
                 fieldspec='1',
                 cache_dir=cache_dir,
             )
 
             next_date_fname_sorted = gnu_sort(
-                parse_and_filter_file(next_date_fname, parser=parser, cache_dir=cache_dir),
+                parse_and_filter_file(next_date_fname, parser=parser, cache_dir=cache_dir),  # type: ignore
                 delimiter=',',
                 fieldspec='1',
                 cache_dir=cache_dir,
             )
         else:
             prev_date_fname_sorted = parse_and_filter_file(
-                prev_date_fname,
+                prev_date_fname,  # type: ignore
                 parser=parser,
                 cache_dir=cache_dir,
             )
             next_date_fname_sorted = parse_and_filter_file(
-                next_date_fname,
+                next_date_fname,  # type: ignore
                 parser=parser,
                 cache_dir=cache_dir,
             )
@@ -155,7 +176,10 @@ class Consistency(data_models.DataModel):
                             yield cls('DARK', path)
 
 
-def _try_to_advance(it, default=None):
+def _try_to_advance(
+        it: 'SupportsNext[str]',
+        default: Optional[str] = None
+) -> Optional[str]:
     try:
         el = next(it)
     except StopIteration:
@@ -163,21 +187,29 @@ def _try_to_advance(it, default=None):
     return el.strip()
 
 
-def min_value(*values):
+def min_value(*values: Optional[str]) -> str:
     '''
     Minimum between the input values, ignoring None
     '''
-    values = [value for value in values if value is not None]
-    if len(values) == 0:
+    values_without_none = cast(list[str], [value for value in values if value is not None])
+    if len(values_without_none) == 0:
         raise ValueError("Input contains 0 non-null values.")
-    return min(values)
+    return min(values_without_none)
 
 
-def split_if_not_none(value, sep=',', fields=2):
+def split_if_not_none(
+        value: Optional[str],
+        sep: str = ',',
+        fields: int = 2
+) -> Union[str, list]:
     return value.split(sep) if value is not None else ([None] * fields)
 
 
-def compare3(it0, it1, it2):
+def compare3(
+    it0: 'Iterable[str]',
+    it1: 'Iterable[str]',
+    it2: 'Iterable[str]'
+) -> 'Iterator[tuple[str, tuple[bool, bool, bool], tuple[Optional[str], Optional[str]]]]':
     '''
     Generator to compare 3 sorted iterables, in each
     iteration it yields a tuple of the form (current, (bool, bool, bool))
@@ -240,7 +272,14 @@ def compare3(it0, it1, it2):
             path2, status2 = split_if_not_none(v2)
 
 
-def parse_and_filter_file(filepath, parser=lambda s: s, filter_=lambda s: s, prefix=None, postfix='parsed', cache_dir=DUMPS_CACHE_DIR):
+def parse_and_filter_file(
+        filepath: str,
+        parser: 'Callable' = lambda s: s,
+        filter_: 'Callable' = lambda s: s,
+        prefix: Optional[str] = None,
+        postfix: str = 'parsed',
+        cache_dir: str = DUMPS_CACHE_DIR
+) -> str:
     '''
     Opens `filepath` as a read-only file, and for each line of the file
     for which the `filter_` function returns True, it writes a version
@@ -286,7 +325,13 @@ def parse_and_filter_file(filepath, parser=lambda s: s, filter_=lambda s: s, pre
     return output_path
 
 
-def gnu_sort(file_path, prefix=None, delimiter=None, fieldspec=None, cache_dir=DUMPS_CACHE_DIR):
+def gnu_sort(
+        file_path: str,
+        prefix: Optional[str] = None,
+        delimiter: Optional[str] = None,
+        fieldspec: Optional[str] = None,
+        cache_dir: str = DUMPS_CACHE_DIR
+) -> str:
     '''
     Sort the file with path `file_path` using the GNU sort command, the
     original file is unchanged, the output file is saved with path
@@ -332,7 +377,7 @@ def gnu_sort(file_path, prefix=None, delimiter=None, fieldspec=None, cache_dir=D
     return sorted_path
 
 
-def populate_args(argparser):
+def populate_args(argparser: '_SubParsersAction') -> None:
     # Option to download the rucio replica dumps automatically
     parser = argparser.add_parser(
         'consistency',
@@ -383,7 +428,7 @@ def populate_args(argparser):
 _date_re = re.compile(r'dump_(\d{8})')
 
 
-def _parse_args_consistency(args):
+def _parse_args_consistency(args: 'Namespace') -> dict[str, datetime.datetime]:
     args_dict = {}
 
     # Filename should contain the date
@@ -392,14 +437,14 @@ def _parse_args_consistency(args):
         error('The storage dump filename must be of the form '
               '"dump_YYYYMMDD" where the date correspond to the date '
               'of the newest files included')
-
-    date_str = date_str.group(1)
-    if date_str is None:
-        error('Invalid date {0}'.format(date_str))
-    try:
-        args_dict['date'] = date = datetime.datetime.strptime(date_str, '%Y%m%d')
-    except ValueError:
-        error('Invalid date {0}'.format(date_str))
+    else:
+        date_str = date_str.group(1)
+        if date_str is None:
+            error('Invalid date {0}'.format(date_str))
+        try:
+            args_dict['date'] = date = datetime.datetime.strptime(date_str, '%Y%m%d')
+        except ValueError:
+            error('Invalid date {0}'.format(date_str))
 
     if not os.path.exists(args.storage_dump):
         error('File "{0}" does not exist'.format(args.storage_dump))
@@ -426,7 +471,7 @@ def _parse_args_consistency(args):
     return args_dict
 
 
-def _parse_args_consistency_manual(args):
+def _parse_args_consistency_manual(args: 'Namespace') -> dict[str, Any]:
     args_dict = {}
     args_dict['prev_date_fname'] = args.replicas_before
     args_dict['next_date_fname'] = args.replicas_after
@@ -438,7 +483,7 @@ def _parse_args_consistency_manual(args):
     return args_dict
 
 
-def parse_args(args):
+def parse_args(args: 'Namespace') -> dict[str, Any]:
     args_dict = {}
     args_dict['subcommand'] = args.subcommand
     args_dict['ddm_endpoint'] = args.ddm_endpoint