rucio 35.0.1__tar.gz → 35.1.0__tar.gz

{rucio-35.0.1 → rucio-35.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: rucio
-Version: 35.0.1
+Version: 35.1.0
 Summary: Rucio Package
 Home-page: https://rucio.cern.ch/
 Author: Rucio

{rucio-35.0.1 → rucio-35.1.0}/bin/rucio-admin

@@ -1196,7 +1196,9 @@ def declare_temporary_unavailable_replicas(args):
     else:
         bad_files = args.listbadfiles
 
-    assert args.duration is not None, "Duration should have been set, something went wrong!"
+    if args.duration is None:
+        raise ValueError("Duration should have been set, something went wrong!")
+
     expiration_date = (datetime.datetime.utcnow() + datetime.timedelta(seconds=args.duration)).isoformat()
 
     chunk_size = 10000

{rucio-35.0.1 → rucio-35.1.0}/bin/rucio-auditor

@@ -21,6 +21,7 @@ import signal
 import sys
 import textwrap
 import time
+from configparser import NoSectionError
 from datetime import datetime
 from functools import partial
 from multiprocessing import Event, Pipe, Process, Queue
@@ -29,6 +30,7 @@ import rucio.common.config as config
 import rucio.common.dumper as dumper
 import rucio.daemons.auditor
 from rucio.client.rseclient import RSEClient
+from rucio.common.exception import RSENotFound
 
 
 def setup_pipe_logger(pipe, loglevel):
@@ -48,14 +50,17 @@ def main(args):
     RETRY_AFTER = 60 * 60 * 24 * 14  # Two weeks
 
     nprocs = args.nprocs
-    assert nprocs >= 1
+    if nprocs < 1:
+        raise RuntimeError("No Processes to Run")
+
     if args.rses is None:
         rses_gen = RSEClient().list_rses()
     else:
         rses_gen = RSEClient().list_rses(args.rses)
 
     rses = [entry['rse'] for entry in rses_gen]
-    assert len(rses) > 0
+    if len(rses) <= 0:
+        raise RSENotFound("No RSEs found to audit.")
 
     procs = []
     queue = Queue()
@@ -69,7 +74,9 @@ def main(args):
     logpipes.append(mainlogr)
     logger = setup_pipe_logger(mainlogw, loglevel)
 
-    assert config.config_has_section('auditor')
+    if not config.config_has_section('auditor'):
+        raise NoSectionError("Auditor section required in config to run the auditor daemon.")
+
     cache_dir = config.config_get('auditor', 'cache')
     results_dir = config.config_get('auditor', 'results')
 

{rucio-35.0.1 → rucio-35.1.0}/bin/rucio-dumper

@@ -95,9 +95,10 @@ if __name__ == "__main__":
         record_type = data_models.CompleteDataset
     elif args.subcommand == 'dump-replicas':
         record_type = data_models.Replica
-    else:
-        assert args.subcommand in consistency.subcommands
+    elif args.subcommand in consistency.subcommands:
         record_type = consistency.Consistency
+    else:
+        raise ValueError("Record type not set! Cannot dump files.")
 
     if 'filter' in args and args.filter:
         user_filter = data_models.Filter(args.filter, record_type).match
@@ -113,8 +114,8 @@ if __name__ == "__main__":
     fields = set(record_type.get_fieldnames())
 
     if 'group_by' in args and args.group_by:
-        # FIXME: Better checking
-        assert args.group_by in fields
+        if args.group_by not in fields:
+            raise ValueError("Group_by option %s not supported." % args.group_by)
         fields = [args.group_by]
         data_iter = data
         data_dict = {}
@@ -125,7 +126,8 @@ if __name__ == "__main__":
             data_dict[key].append(record)
 
         if 'sum' in args and args.sum:
-            assert args.sum in set(record_type.get_fieldnames())
+            if args.sum not in set(record_type.get_fieldnames()):
+                raise ValueError("Sum option %s for record type %s not supported.\nChoice from %s" % (args.sum, args.record_type, set(record_type.get_fieldnames())))
             fields.append(args.sum)
             data = []
             for key, value in data_dict.items():

{rucio-35.0.1 → rucio-35.1.0}/lib/rucio/client/lockclient.py

@@ -69,8 +69,8 @@ class LockClient(BaseClient):
         """
 
         # convert did list to list of dictionaries
-
-        assert all(did.get("type", "dataset") in ("dataset", "container") for did in dids), "did type can be either 'container' or 'dataset'"
+        if not all(did.get("type", "dataset") in ("dataset", "container") for did in dids):
+            raise ValueError("DID type can be either 'container' or 'dataset'")
 
         path = '/'.join([self.LOCKS_BASEURL, "bulk_locks_for_dids"])
         url = build_url(choice(self.list_hosts), path=path)

{rucio-35.0.1 → rucio-35.1.0}/lib/rucio/client/uploadclient.py

@@ -48,7 +48,7 @@ from rucio.rse import rsemanager as rsemgr
 if TYPE_CHECKING:
     from collections.abc import Iterable, Mapping
 
-    from rucio.common.types import AttachDict, DatasetDict, DIDStringDict, FileToUploadDict, FileToUploadWithCollectedAndDatasetInfoDict, FileToUploadWithCollectedInfoDict, LoggerFunction, RSESettingsDict, TraceBaseDict, TraceDict
+    from rucio.common.types import AttachDict, DatasetDict, DIDStringDict, FileToUploadDict, FileToUploadWithCollectedAndDatasetInfoDict, FileToUploadWithCollectedInfoDict, LoggerFunction, PathTypeAlias, RSESettingsDict, TraceBaseDict, TraceDict
     from rucio.rse.protocols.protocol import RSEProtocol
 
 
@@ -372,10 +372,7 @@ class UploadClient:
             raise NotAllFilesUploaded()
         return 0
 
-    def _add_bittorrent_meta(self, file: "Mapping[str, Any]", logger: "LoggerFunction") -> None:
-        if not config_get_bool('client', 'register_bittorrent_meta', default=False):
-            return
-
+    def _add_bittorrent_meta(self, file: "Mapping[str, Any]") -> None:
         pieces_root, pieces_layers, piece_length = bittorrent_v2_merkle_sha256(os.path.join(file['dirname'], file['basename']))
         bittorrent_meta = {
             'bittorrent_pieces_root': base64.b64encode(pieces_root).decode(),
@@ -383,6 +380,7 @@ class UploadClient:
             'bittorrent_piece_length': piece_length,
         }
         self.client.set_metadata_bulk(scope=file['did_scope'], name=file['did_name'], meta=bittorrent_meta)
+        self.logger(logging.INFO, f"Added bittorrent metadata to file DID {file['did_scope']}:{file['did_name']}")
 
     def _register_file(
             self,
@@ -434,7 +432,6 @@ class UploadClient:
                 logger(logging.INFO, 'Successfully created dataset %s' % dataset_did_str)
             except DataIdentifierAlreadyExists:
                 logger(logging.INFO, 'Dataset %s already exists - no rule will be created' % dataset_did_str)
-
                 if file.get('lifetime') is not None:
                     raise InputValidationError('Dataset %s exists and lifetime %s given. Prohibited to modify parent dataset lifetime.' % (dataset_did_str,
                                                                                                                                            file.get('lifetime')))
@@ -453,7 +450,6 @@ class UploadClient:
 
             if str(meta['adler32']).lstrip('0') != str(file['adler32']).lstrip('0'):
                 logger(logging.ERROR, 'Local checksum %s does not match remote checksum %s' % (file['adler32'], meta['adler32']))
-
                 raise DataIdentifierAlreadyExists
 
             # add file to rse if it is not registered yet
@@ -464,7 +460,8 @@ class UploadClient:
         except DataIdentifierNotFound:
             logger(logging.DEBUG, 'File DID does not exist')
             self.client.add_replicas(rse=rse, files=[replica_for_api])
-            self._add_bittorrent_meta(file=file, logger=logger)
+            if config_get_bool('client', 'register_bittorrent_meta', default=False):
+                self._add_bittorrent_meta(file=file)
             logger(logging.INFO, 'Successfully added replica in Rucio catalogue at %s' % rse)
             if not dataset_did_str:
                 # only need to add rules for files if no dataset is given
@@ -499,7 +496,7 @@ class UploadClient:
 
     def _collect_file_info(
             self,
-            filepath: str,
+            filepath: "PathTypeAlias",
             item: "FileToUploadDict"
     ) -> "FileToUploadWithCollectedInfoDict":
         """
@@ -845,7 +842,7 @@ class UploadClient:
             scope = self.default_file_scope
         rse = item.get('rse')
         path = item.get('path')
-        if path:
+        if path and isinstance(path, str):
             if path[-1] == '/':
                 path = path[0:-1]
             i = 0

{rucio-35.0.1 → rucio-35.1.0}/lib/rucio/common/dumper/__init__.py

@@ -81,7 +81,8 @@ def mkdir(dir_: "StrOrBytesPath") -> None:
     try:
         os.mkdir(dir_)
     except OSError as error:
-        assert error.errno == 17
+        if error.errno != 17:
+            raise error
 
 
 def cacert_config(config: "ModuleType", rucio_home: str) -> Optional[Union["FileDescriptorOrPath", Literal[False]]]:

{rucio-35.0.1 → rucio-35.1.0}/lib/rucio/common/dumper/consistency.py

@@ -41,10 +41,12 @@ class Consistency(data_models.DataModel):
                 ddm_endpoint, prev_date, cache_dir=cache_dir)
             next_date_fname = data_models.Replica.download(
                 ddm_endpoint, next_date, cache_dir=cache_dir)
-            assert prev_date_fname is not None
-            assert next_date_fname is not None
+            if (prev_date_fname is None) or (next_date_fname is None):
+                raise ValueError("Both prev_date_fname and next_date_fname are required for subcommand='consistency'; found prev_data_fname=%s and next_date_fname=%s" % (prev_date_fname, next_date_fname))
+        elif subcommand == 'consistency-manual':
+            pass
         else:
-            assert subcommand == 'consistency-manual'
+            raise ValueError("subcommand %s not accepted, choice from ('consistency','consistency-manual')" % subcommand)
 
         prefix_components = path_parsing.components(dumper.ddmendpoint_url(ddm_endpoint))
 
@@ -166,7 +168,8 @@ def min3(*values):
     Minimum between the 3 values ignoring None
     '''
     values = [value for value in values if value is not None]
-    assert len(values) > 0
+    if len(values) == 0:
+        raise ValueError("Input contains 0 non-null values.")
     return min(values)
 
 
@@ -301,7 +304,8 @@ def gnu_sort(file_path, prefix=None, delimiter=None, fieldspec=None, cache_dir=D
     memory and it is relatively fast if used with the environment variable
     LC_ALL set to C as in this function.
     '''
-    assert (delimiter is None and fieldspec is None) or (delimiter is not None and fieldspec is not None)
+    if (delimiter is not None) ^ (fieldspec is not None):
+        raise ValueError("Either both delimiter and fieldspec is set, or neither are.")
     if delimiter is None:
         cmd_line = 'LC_ALL=C sort {0} > {1}'
     else:
@@ -388,8 +392,10 @@ def _parse_args_consistency(args):
         error('The storage dump filename must be of the form '
               '"dump_YYYYMMDD" where the date correspond to the date '
               'of the newest files included')
+
     date_str = date_str.group(1)
-    assert date_str is not None
+    if date_str is None:
+        error('Invalid date {0}'.format(date_str))
     try:
         args_dict['date'] = date = datetime.datetime.strptime(date_str, '%Y%m%d')
     except ValueError:

{rucio-35.0.1 → rucio-35.1.0}/lib/rucio/common/dumper/data_models.py

@@ -150,20 +150,17 @@ class DataModel:
         logger = logging.getLogger('auditor.data_models')
         requests_session = get_requests_session()
         if date == 'latest':
-            url = ''.join((cls.BASE_URL, cls.URI, '?rse={0}'.format(rse)))
+            url = ''.join((cls.BASE_URL, cls.URI, '?rse={0}'.format(rse)))  # type: ignore
             request_headers = requests_session.head(url)
             for field in request_headers.headers['content-disposition'].split(';'):
                 if field.startswith('filename='):
                     date = field.split('=')[1].split('_')[-1].split('.')[0]
 
+        elif isinstance(date, datetime.datetime):
+            date = date.strftime('%d-%m-%Y')
+            url = ''.join((cls.BASE_URL, cls.URI, '?rse={0}&date={1}'.format(rse, date)))  # type: ignore
         else:
-            assert isinstance(date, datetime.datetime)
-            date = date.strftime('%d-%m-%Y')  # pylint: disable=no-member
-            url = ''.join((
-                cls.BASE_URL,
-                cls.URI,
-                '?rse={0}&date={1}'.format(rse, date),
-            ))
+            raise ValueError("Passed date (%s) must be a datetime object or 'latest'." % date)
 
         if not os.path.isdir(cache_dir):
             os.mkdir(cache_dir)
@@ -237,7 +234,8 @@ class CompleteDataset(DataModel):
             self.state = args[7]
         else:
             self.state = None
-        assert len(args) <= 8
+        if len(args) > 8:
+            raise ValueError("Too many arguments, must be 8 or less. Instead passed %s" % len(args))
 
 
 class Replica(DataModel):
@@ -264,7 +262,8 @@ class Replica(DataModel):
 
         if len(args) == 8:
             logger.warning('Missing parameter\nrse: %s\ndataset: %s\n', self.rse, self.name)
-        assert len(args) <= 9
+        elif len(args) > 9:
+            raise ValueError("Too many arguments. Must be 9 or less, instead passed %s" % len(args))
 
 
 class Filter:
@@ -292,7 +291,8 @@ class Filter:
         for expr in filter_str.split(','):
             key, expected = expr.split('=')
             # Better checks required
-            assert key in record_class.get_fieldnames()
+            if key not in record_class.get_fieldnames():
+                raise ValueError("Key %s not supported." % key)
             parser = list(filter(lambda t: t[0] == key, record_class.SCHEMA))[0][1]
             self.conditions.append(self._Condition(
                 comparator=operator.eq,

{rucio-35.0.1 → rucio-35.1.0}/lib/rucio/common/types.py

@@ -15,11 +15,14 @@
 import sys
 from collections.abc import Callable
 from datetime import datetime
+from os import PathLike
 
 if sys.version_info < (3, 11):
     from typing_extensions import TYPE_CHECKING, Any, Literal, NotRequired, Optional, TypedDict, Union  # noqa: UP035
+    PathTypeAlias = Union[PathLike, str]
 else:
     from typing import TYPE_CHECKING, Any, Literal, NotRequired, Optional, TypedDict, Union
+    PathTypeAlias = PathLike
 
 
 if TYPE_CHECKING:
@@ -306,7 +309,7 @@ class TraceDict(TraceBaseDict):
 
 
 class FileToUploadDict(TypedDict):
-    path: str
+    path: PathTypeAlias
     rse: str
     did_scope: str
     did_name: str

{rucio-35.0.1 → rucio-35.1.0}/lib/rucio/common/utils.py

@@ -1941,7 +1941,8 @@ class StoreAndDeprecateWarningAction(argparse.Action):
             option_strings=option_strings,
             dest=dest,
             **kwargs)
-        assert new_option_string in option_strings
+        if new_option_string not in option_strings:
+            raise ValueError("%s not supported as a string option." % new_option_string)
         self.new_option_string = new_option_string
 
     def __call__(self, parser, namespace, values, option_string: Optional[str] = None):
@@ -1979,7 +1980,8 @@ class StoreTrueAndDeprecateWarningAction(argparse._StoreConstAction):
             default=default,
             required=required,
             help=help)
-        assert new_option_string in option_strings
+        if new_option_string not in option_strings:
+            raise ValueError("%s not supported as a string option." % new_option_string)
         self.new_option_string = new_option_string
 
     def __call__(self, parser, namespace, values, option_string: Optional[str] = None):