rucio 35.7.0__py3-none-any.whl → 37.0.0__py3-none-any.whl

rucio/common/dumper/consistency.py

@@ -18,10 +18,17 @@ import os
 import re
 import subprocess
 import tempfile
+from typing import TYPE_CHECKING, Any, Optional, Union, cast
 
 from rucio.common import dumper
 from rucio.common.dumper import DUMPS_CACHE_DIR, data_models, error, path_parsing
 
+if TYPE_CHECKING:
+    from argparse import Namespace, _SubParsersAction
+    from collections.abc import Callable, Iterable, Iterator
+
+    from _typeshed import SupportsNext
+
 subcommands = ['consistency', 'consistency-manual']
 
 
@@ -32,11 +39,25 @@ class Consistency(data_models.DataModel):
     )
 
     @classmethod
-    def dump(cls, subcommand, ddm_endpoint, storage_dump, prev_date_fname=None, next_date_fname=None,
-             prev_date=None, next_date=None, sort_rucio_replica_dumps=True, date=None,
-             cache_dir=DUMPS_CACHE_DIR):
+    def dump(
+        cls,
+        subcommand: str,
+        ddm_endpoint: str,
+        storage_dump: str,
+        prev_date_fname: Optional[str] = None,
+        next_date_fname: Optional[str] = None,
+        prev_date: Optional[Union[str, datetime.datetime]] = None,
+        next_date: Optional[Union[str, datetime.datetime]] = None,
+        sort_rucio_replica_dumps: bool = True,
+        date: Optional[datetime.datetime] = None,
+        cache_dir: str = DUMPS_CACHE_DIR
+    ):
         logger = logging.getLogger('auditor.consistency')
         if subcommand == 'consistency':
+            if prev_date is None:
+                prev_date = 'latest'
+            if next_date is None:
+                next_date = 'latest'
             prev_date_fname = data_models.Replica.download(
                 ddm_endpoint, prev_date, cache_dir=cache_dir)
             next_date_fname = data_models.Replica.download(
@@ -50,7 +71,7 @@ class Consistency(data_models.DataModel):
 
         prefix_components = path_parsing.components(dumper.ddmendpoint_url(ddm_endpoint))
 
-        def parser(line):
+        def parser(line: str) -> str:
             '''
             Simple parser for Rucio replica dumps.
 
@@ -63,7 +84,7 @@ class Consistency(data_models.DataModel):
 
             return ','.join((path, status))
 
-        def strip_storage_dump(line):
+        def strip_storage_dump(line: str) -> str:
             '''
             Parser to have consistent paths in storage dumps.
 
@@ -80,26 +101,26 @@ class Consistency(data_models.DataModel):
 
         if sort_rucio_replica_dumps:
             prev_date_fname_sorted = gnu_sort(
-                parse_and_filter_file(prev_date_fname, parser=parser, cache_dir=cache_dir),
+                parse_and_filter_file(prev_date_fname, parser=parser, cache_dir=cache_dir),  # type: ignore
                 delimiter=',',
                 fieldspec='1',
                 cache_dir=cache_dir,
             )
 
             next_date_fname_sorted = gnu_sort(
-                parse_and_filter_file(next_date_fname, parser=parser, cache_dir=cache_dir),
+                parse_and_filter_file(next_date_fname, parser=parser, cache_dir=cache_dir),  # type: ignore
                 delimiter=',',
                 fieldspec='1',
                 cache_dir=cache_dir,
             )
         else:
             prev_date_fname_sorted = parse_and_filter_file(
-                prev_date_fname,
+                prev_date_fname,  # type: ignore
                 parser=parser,
                 cache_dir=cache_dir,
             )
             next_date_fname_sorted = parse_and_filter_file(
-                next_date_fname,
+                next_date_fname,  # type: ignore
                 parser=parser,
                 cache_dir=cache_dir,
             )
@@ -155,7 +176,10 @@ class Consistency(data_models.DataModel):
                             yield cls('DARK', path)
 
 
-def _try_to_advance(it, default=None):
+def _try_to_advance(
+        it: 'SupportsNext[str]',
+        default: Optional[str] = None
+) -> Optional[str]:
     try:
         el = next(it)
     except StopIteration:
@@ -163,21 +187,29 @@ def _try_to_advance(it, default=None):
     return el.strip()
 
 
-def min_value(*values):
+def min_value(*values: Optional[str]) -> str:
     '''
     Minimum between the input values, ignoring None
     '''
-    values = [value for value in values if value is not None]
-    if len(values) == 0:
+    values_without_none = cast('list[str]', [value for value in values if value is not None])
+    if len(values_without_none) == 0:
         raise ValueError("Input contains 0 non-null values.")
-    return min(values)
+    return min(values_without_none)
 
 
-def split_if_not_none(value, sep=',', fields=2):
+def split_if_not_none(
+        value: Optional[str],
+        sep: str = ',',
+        fields: int = 2
+) -> Union[str, list]:
     return value.split(sep) if value is not None else ([None] * fields)
 
 
-def compare3(it0, it1, it2):
+def compare3(
+    it0: 'Iterable[str]',
+    it1: 'Iterable[str]',
+    it2: 'Iterable[str]'
+) -> 'Iterator[tuple[str, tuple[bool, bool, bool], tuple[Optional[str], Optional[str]]]]':
     '''
     Generator to compare 3 sorted iterables, in each
     iteration it yields a tuple of the form (current, (bool, bool, bool))
@@ -240,7 +272,14 @@ def compare3(it0, it1, it2):
             path2, status2 = split_if_not_none(v2)
 
 
-def parse_and_filter_file(filepath, parser=lambda s: s, filter_=lambda s: s, prefix=None, postfix='parsed', cache_dir=DUMPS_CACHE_DIR):
+def parse_and_filter_file(
+        filepath: str,
+        parser: 'Callable' = lambda s: s,
+        filter_: 'Callable' = lambda s: s,
+        prefix: Optional[str] = None,
+        postfix: str = 'parsed',
+        cache_dir: str = DUMPS_CACHE_DIR
+) -> str:
     '''
     Opens `filepath` as a read-only file, and for each line of the file
     for which the `filter_` function returns True, it writes a version
@@ -286,7 +325,13 @@ def parse_and_filter_file(filepath, parser=lambda s: s, filter_=lambda s: s, pre
     return output_path
 
 
-def gnu_sort(file_path, prefix=None, delimiter=None, fieldspec=None, cache_dir=DUMPS_CACHE_DIR):
+def gnu_sort(
+        file_path: str,
+        prefix: Optional[str] = None,
+        delimiter: Optional[str] = None,
+        fieldspec: Optional[str] = None,
+        cache_dir: str = DUMPS_CACHE_DIR
+) -> str:
     '''
     Sort the file with path `file_path` using the GNU sort command, the
     original file is unchanged, the output file is saved with path
@@ -332,7 +377,7 @@ def gnu_sort(file_path, prefix=None, delimiter=None, fieldspec=None, cache_dir=D
     return sorted_path
 
 
-def populate_args(argparser):
+def populate_args(argparser: '_SubParsersAction') -> None:
     # Option to download the rucio replica dumps automatically
     parser = argparser.add_parser(
         'consistency',
@@ -383,7 +428,7 @@ def populate_args(argparser):
 _date_re = re.compile(r'dump_(\d{8})')
 
 
-def _parse_args_consistency(args):
+def _parse_args_consistency(args: 'Namespace') -> dict[str, datetime.datetime]:
     args_dict = {}
 
     # Filename should contain the date
@@ -392,14 +437,14 @@ def _parse_args_consistency(args):
         error('The storage dump filename must be of the form '
               '"dump_YYYYMMDD" where the date correspond to the date '
               'of the newest files included')
-
-    date_str = date_str.group(1)
-    if date_str is None:
-        error('Invalid date {0}'.format(date_str))
-    try:
-        args_dict['date'] = date = datetime.datetime.strptime(date_str, '%Y%m%d')
-    except ValueError:
-        error('Invalid date {0}'.format(date_str))
+    else:
+        date_str = date_str.group(1)
+        if date_str is None:
+            error('Invalid date {0}'.format(date_str))
+        try:
+            args_dict['date'] = date = datetime.datetime.strptime(date_str, '%Y%m%d')
+        except ValueError:
+            error('Invalid date {0}'.format(date_str))
 
     if not os.path.exists(args.storage_dump):
         error('File "{0}" does not exist'.format(args.storage_dump))
@@ -426,7 +471,7 @@ def _parse_args_consistency(args):
     return args_dict
 
 
-def _parse_args_consistency_manual(args):
+def _parse_args_consistency_manual(args: 'Namespace') -> dict[str, Any]:
     args_dict = {}
     args_dict['prev_date_fname'] = args.replicas_before
     args_dict['next_date_fname'] = args.replicas_after
@@ -438,7 +483,7 @@ def _parse_args_consistency_manual(args):
     return args_dict
 
 
-def parse_args(args):
+def parse_args(args: 'Namespace') -> dict[str, Any]:
     args_dict = {}
     args_dict['subcommand'] = args.subcommand
     args_dict['ddm_endpoint'] = args.ddm_endpoint

rucio/common/dumper/data_models.py

@@ -23,11 +23,15 @@ import logging
 import operator
 import os
 import re
+from typing import TYPE_CHECKING, Any, Optional, TextIO, Union
 
 from tabulate import tabulate
 
 from rucio.common.dumper import DUMPS_CACHE_DIR, HTTPDownloadFailed, get_requests_session, http_download_to_file, smart_open, temp_file, to_datetime
 
+if TYPE_CHECKING:
+    from collections.abc import Callable, Iterable, Iterator
+
 
 class DataModel:
     """
@@ -35,12 +39,12 @@ class DataModel:
     """
 
     BASE_URL = 'https://rucio-hadoop.cern.ch/'
-    _FIELD_NAMES = None
+    _FIELD_NAMES: Optional[list[str]] = None
     SCHEMA = []
     URI = None
     name = None
 
-    def __init__(self, *args):
+    def __init__(self, *args) -> None:
         if len(args) != len(self.SCHEMA):
             raise TypeError(
                 'Wrong number of parameters (fields) to initialize {0} '
@@ -62,6 +66,7 @@ class DataModel:
                 raise err
 
         self.date = None
+        self.rse = None
 
     @classmethod
     def get_fieldnames(cls) -> list[str]:
@@ -72,7 +77,7 @@ class DataModel:
             cls._FIELD_NAMES = [name for name, _ in cls.SCHEMA]
         return cls._FIELD_NAMES
 
-    def pprint(self):
+    def pprint(self) -> str:
         """
         Pretty print the dump
         """
@@ -80,14 +85,17 @@ class DataModel:
             ['{0}: {1}\n'.format(attr, getattr(self, attr)) for attr, _ in self.SCHEMA]
         )
 
-    def __getitem__(self, index):
+    def __getitem__(self, index: int) -> Any:
         """
         Return the item
         """
         return getattr(self, self.SCHEMA[index][0])
 
     @classmethod
-    def csv_header(cls, fields=None):
+    def csv_header(
+        cls,
+        fields: Optional["Iterable[str]"] = None
+    ) -> str:
         """
         Add the CSV header if necessary
         """
@@ -95,7 +103,10 @@ class DataModel:
             fields = (field for field, _ in cls.SCHEMA)
         return ','.join(fields)
 
-    def formated_fields(self, print_fields=None):
+    def formated_fields(
+        self,
+        print_fields: Optional["Iterable[str]"] = None
+    ) -> list[str]:
         """
         Reformat the fields
         """
@@ -110,31 +121,54 @@ class DataModel:
             fields.append(str(field))
         return fields
 
-    def csv(self, fields=None):
+    def csv(
+        self,
+        fields: Optional["Iterable[str]"] = None
+    ) -> str:
         """
         Generate a CSV line
         """
         return ','.join(self.formated_fields(fields))
 
     @classmethod
-    def tabulate_from(cls, iter_, format_='simple', fields=None):
+    def tabulate_from(
+        cls,
+        iter_: "Iterable[DataModel]",
+        format_: str = 'simple',
+        fields: Optional["Iterable[str]"] = None
+    ) -> str:
         return tabulate(
             (row.formated_fields(fields) for row in iter_),
-            (t[0] for t in cls.SCHEMA),
+            (t[0] for t in cls.SCHEMA),  # type: ignore
             format_,
         )
 
     @classmethod
-    def each(cls, file, rse=None, date=None, filter_=None):
+    def each(
+        cls,
+        file: "TextIO",
+        rse: Optional[str] = None,
+        date: Optional[Union[str, datetime.datetime]] = None,
+        filter_: Optional["Callable"] = None
+    ) -> "Iterator[DataModel]":
         if filter_ is None:
-            filter_ = lambda x: True  # NOQA
+
+            def placeholder_filter(record: "DataModel") -> bool:
+                return True
+
+            filter_ = placeholder_filter
         for line in file:
             record = cls.parse_line(line, rse, date)
             if filter_(record):
                 yield record
 
     @classmethod
-    def parse_line(cls, line, rse=None, date=None):
+    def parse_line(
+        cls,
+        line: str,
+        rse: Optional[str] = None,
+        date: Optional[Union[str, datetime.datetime]] = None
+    ) -> "DataModel":
         fields = (field.strip() for field in line.split('\t'))
         instance = cls(*fields)
         instance.rse = rse
@@ -142,7 +176,12 @@ class DataModel:
         return instance
 
     @classmethod
-    def download(cls, rse, date='latest', cache_dir=DUMPS_CACHE_DIR):
+    def download(
+        cls,
+        rse: str,
+        date: Union[str, datetime.datetime] = 'latest',
+        cache_dir: str = DUMPS_CACHE_DIR
+    ) -> str:
         """
         Downloads the requested dump and returns an open read-only mode file
         like object.
@@ -191,13 +230,18 @@ class DataModel:
         return path
 
     @classmethod
-    def dump(cls, rse, date='latest', filter_=None):
+    def dump(
+        cls,
+        rse: str,
+        date: Union[str, datetime.datetime] = 'latest',
+        filter_: Optional["Callable"] = None
+    ) -> "Iterator[DataModel]":
         filename = cls.download(rse, date)
 
         # Should check errors, content size at least
         file = smart_open(filename)
 
-        return cls.each(file, rse, date, filter_)
+        return cls.each(file, rse, date, filter_)  # type: ignore (file could be None)
 
 
 class Dataset(DataModel):
@@ -226,7 +270,7 @@ class CompleteDataset(DataModel):
         ('last_access', to_datetime),
     )
 
-    def __init__(self, *args):
+    def __init__(self, *args) -> None:
         logger = logging.getLogger('auditor.data_models')
         super(CompleteDataset, self).__init__(*args[0:7])
         if len(args) == 8:
@@ -252,7 +296,7 @@ class Replica(DataModel):
         ('state', str),
     )
 
-    def __init__(self, *args):
+    def __init__(self, *args) -> None:
         logger = logging.getLogger('auditor.data_models')
         if len(args) == 8:
             args = list(args)
@@ -269,7 +313,7 @@ class Replica(DataModel):
 class Filter:
     _Condition = collections.namedtuple('_Condition', ('comparator', 'attribute', 'expected'))
 
-    def __init__(self, filter_str, record_class):
+    def __init__(self, filter_str: str, record_class: DataModel) -> None:
         '''
         Filter objects allow to match a DataModel subclass instance against
         one or more conditions.
@@ -300,7 +344,7 @@ class Filter:
                 expected=parser(expected),
             ))
 
-    def match(self, record):
+    def match(self, record: DataModel) -> bool:
         '''
         :param record: DataModel subclass instance.
         :returns: True if record matches all the conditions in this filter,

rucio/common/dumper/path_parsing.py

@@ -17,15 +17,18 @@ import logging
 logger = logging.getLogger('rucio_dumps')
 
 
-def prefix(agis_data, endpoint_name):
-    ddmendpoint_data = list(filter(
-        lambda d: d['name'] == endpoint_name,
-        agis_data,
-    ))[0]
-    return ddmendpoint_data['endpoint']
+def remove_prefix(prefix: list[str], path: list[str]) -> list[str]:
+    """
+    Remove the specified prefix from the given path.
 
+    :param prefix: The prefix to be removed from the path.
+    :param path: The path from which the prefix should be removed.
+
+    :return: The path with the prefix removed.
+            If the prefix is not found at the start of the path, the original path is returned.
+            If the path is a subset of the prefix, an empty list is returned.
+    """
 
-def remove_prefix(prefix, path):
     iprefix = iter(prefix)
     ipath = iter(path)
     try:
@@ -59,6 +62,14 @@ def remove_prefix(prefix, path):
     return rest
 
 
-def components(path):
+def components(path: str) -> list[str]:
+    """
+    Extracts and returns the non-empty components of a given path.
+
+    :param path: input path string to be parsed.
+
+    :return: list of non-empty components of the path.
+    """
+
     components = path.strip().strip('/').split('/')
     return [component for component in components if component != '']

rucio/common/exception.py

@@ -20,6 +20,8 @@
 
 """
 
+from typing import Optional
+
 from rucio.common.constraints import AUTHORIZED_VALUE_TYPES
 
 
@@ -93,11 +95,12 @@ class ClientParameterMismatch(RucioException):
 
 class ClientProtocolNotSupported(RucioException):
     """
-    RucioException
+    Client protocol not supported
     """
-    def __init__(self, *args):
+
+    def __init__(self, host: str, protocol: str, protocols_allowed: Optional[list[str]] = None, *args):
         super(ClientProtocolNotSupported, self).__init__(*args)
-        self._message = "Client protocol not supported."
+        self._message = f"Client protocol '{protocol}' not supported when connecting to host '{host}'.{' Allowed protocols: ' + ', '.join(protocols_allowed) if protocols_allowed else ''}"
         self.error_code = 6
 
 
@@ -1068,14 +1071,14 @@ class PolicyPackageVersionError(PolicyPackageBaseException):
     """
     Policy package is not compatible with this version of Rucio.
     """
-    def __init__(self, package: str, rucio_version: str, supported_versions: list[str], *args):
+    def __init__(self, package: str, rucio_version: str, supported_versionset: str, *args):
         super(PolicyPackageVersionError, self).__init__(package, *args)
         self.rucio_version = rucio_version
-        self.supported_versions = supported_versions
+        self.supported_versionset = supported_versionset
         self._message = 'Policy package %s is not compatible with this Rucio version.\nRucio version: %s\nVersions supported by the package: %s' % (
             self.package,
             self.rucio_version,
-            self.supported_versions
+            self.supported_versionset
         )
         self.error_code = 103
 
@@ -1125,8 +1128,8 @@ class TraceValidationSchemaNotFound(RucioException):
     """
     Trace validation schema not found.
     """
-    def __init__(self, *args, **kwargs):
-        super(TraceValidationSchemaNotFound, self).__init__(*args, **kwargs)
+    def __init__(self, *args):
+        super(TraceValidationSchemaNotFound, self).__init__(*args)
         self._message = 'Trace validation schema not found.'
         self.error_code = 108
 
@@ -1149,3 +1152,57 @@ class UnsupportedMetadataPlugin(RucioException):
         super(UnsupportedMetadataPlugin, self).__init__(*args)
         self._message = "The requested metadata plugin is not enabled on the server."
         self.error_code = 110
+
+
+class ChecksumCalculationError(RucioException):
+    """
+    An error occurred while calculating the checksum.
+    """
+    def __init__(
+            self,
+            algorithm_name: str,
+            filepath: str,
+            *args,
+            **kwargs
+    ):
+        super(ChecksumCalculationError, self).__init__(*args, **kwargs)
+        self.algorithm_name = algorithm_name
+        self.filepath = filepath
+        self._message = 'An error occurred while calculating the %s checksum of file %s.' % (self.algorithm_name, self.filepath)
+        self.error_code = 111
+
+
+class ConfigLoadingError(RucioException):
+    """
+    An error occurred while loading the configuration.
+    """
+    def __init__(
+            self,
+            config_file: str,
+            *args,
+            **kwargs
+    ):
+        super(ConfigLoadingError, self).__init__(*args, **kwargs)
+        self._message = 'Could not load Rucio configuration file. Rucio tried loading the following configuration file:\n\t %s' % (config_file)
+        self.error_code = 112
+
+
+class ClientProtocolNotFound(RucioException):
+    """
+    Missing protocol in client configuration (e.g. no http/https in url).
+    """
+
+    def __init__(self, host: str, protocols_allowed: Optional[list[str]] = None, *args):
+        super(ClientProtocolNotFound, self).__init__(*args)
+        self._message = f"Client protocol missing when connecting to host '{host}'.{' Allowed protocols: ' + ', '.join(protocols_allowed) if protocols_allowed else ''}"
+        self.error_code = 113
+
+
+class ConnectionParameterNotFound(RucioException):
+    """
+    Thrown when a required connection parameter is missing.
+    """
+    def __init__(self, param: str, *args):
+        super(ConnectionParameterNotFound, self).__init__(*args)
+        self._message = f"Required connection parameter '{param}' is not provided."
+        self.error_code = 114

rucio/common/extra.py

@@ -13,24 +13,19 @@
 # limitations under the License.
 
 import importlib
-import warnings
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Optional
 
 if TYPE_CHECKING:
-    from typing import Any
+    from collections.abc import Iterable
+    from types import ModuleType
 
 
-def import_extras(module_list: list[str]) -> "dict[str, Any]":
+def import_extras(module_list: 'Iterable[str]') -> dict[str, "Optional[ModuleType]"]:
     out = dict()
     for mod in module_list:
         out[mod] = None
         try:
-            with warnings.catch_warnings():
-                # TODO: remove when https://github.com/paramiko/paramiko/issues/2038 is fixed
-                warnings.filterwarnings('ignore', 'Blowfish has been deprecated', module='paramiko')
-                # TODO: deprecated python 2 and 3.6 too ...
-                warnings.filterwarnings('ignore', 'Python .* is no longer supported', module='paramiko')
-                out[mod] = importlib.import_module(mod)
+            out[mod] = importlib.import_module(mod)
         except ImportError:
             pass
     return out