rucio 35.7.0__py3-none-any.whl → 37.0.0rc2__py3-none-any.whl

rucio/core/topology.py

@@ -17,7 +17,6 @@ import itertools
 import logging
 import threading
 import weakref
-from collections.abc import Callable, Iterable, Iterator
 from decimal import Decimal
 from typing import TYPE_CHECKING, Any, Generic, Optional, TypeVar, Union, cast
 
@@ -38,6 +37,7 @@ TE = TypeVar("TE", bound="Edge")
 ExpiringObjectCacheNewObject = TypeVar("ExpiringObjectCacheNewObject")
 
 if TYPE_CHECKING:
+    from collections.abc import Callable, Iterable, Iterator
     from typing import Protocol
 
     from sqlalchemy.orm import Session
@@ -63,7 +63,7 @@ INF = float('inf')
 
 
 class Node(RseData):
-    def __init__(self, rse_id: str):
+    def __init__(self, rse_id: str) -> None:
         super().__init__(rse_id)
 
         self.in_edges = weakref.WeakKeyDictionary()
@@ -75,7 +75,7 @@ class Node(RseData):
 
 
 class Edge(Generic[TN]):
-    def __init__(self, src_node: TN, dst_node: TN):
+    def __init__(self, src_node: TN, dst_node: TN) -> None:
         self._src_node = weakref.ref(src_node)
         self._dst_node = weakref.ref(dst_node)
 
@@ -123,11 +123,11 @@ class Topology(RseCollection, Generic[TN, TE]):
     """
     def __init__(
             self,
-            rse_ids: Optional[Iterable[str]] = None,
+            rse_ids: Optional["Iterable[str]"] = None,
             ignore_availability: bool = False,
             node_cls: type[TN] = Node,
             edge_cls: type[TE] = Edge,
-    ):
+    ) -> None:
         super().__init__(rse_ids=rse_ids, rse_data_cls=node_cls)
         self._edge_cls = edge_cls
         self._edges: dict[tuple[TN, TN], TE] = {}
@@ -284,7 +284,7 @@ class Topology(RseCollection, Generic[TN, TE]):
     @read_session
     def search_shortest_paths(
             self,
-            src_nodes: Iterable[TN],
+            src_nodes: "Iterable[TN]",
             dst_node: TN,
             operation_src: str,
             operation_dest: str,
@@ -310,7 +310,7 @@ class Topology(RseCollection, Generic[TN, TE]):
         class _NodeStateProvider:
             _hop_penalty = self._hop_penalty
 
-            def __init__(self, node: TN):
+            def __init__(self, node: TN) -> None:
                 self.enabled: bool = True
                 self.cost: _Number = 0
                 if node != dst_node:
@@ -322,7 +322,7 @@ class Topology(RseCollection, Generic[TN, TE]):
         scheme_missmatch_found = {}
 
         class _EdgeStateProvider:
-            def __init__(self, edge: TE):
+            def __init__(self, edge: TE) -> None:
                 self.edge = edge
                 self.chosen_scheme = {}
 
@@ -358,7 +358,7 @@ class Topology(RseCollection, Generic[TN, TE]):
                                                                                  node_state_provider=_NodeStateProvider,
                                                                                  edge_state_provider=_EdgeStateProvider):
             nh_node = edge_to_next_hop.dst_node
-            edge_state = cast(_EdgeStateProvider, edge_state)
+            edge_state = cast("_EdgeStateProvider", edge_state)
             hop = {
                 'source_rse': node,
                 'dest_rse': nh_node,
@@ -433,8 +433,8 @@ class ExpiringObjectCache(Generic[ExpiringObjectCacheNewObject]):
     def __init__(
             self,
             ttl: int,
-            new_obj_fnc: Callable[[], ExpiringObjectCacheNewObject]
-    ):
+            new_obj_fnc: "Callable[[], ExpiringObjectCacheNewObject]"
+    ) -> None:
         self._lock = threading.Lock()
         self._object: Optional[ExpiringObjectCacheNewObject] = None
         self._creation_time: Optional[datetime.datetime] = None

rucio/core/trace.py

@@ -26,7 +26,7 @@ from typing import TYPE_CHECKING, Any, Union, overload
 import stomp
 from jsonschema import Draft7Validator, ValidationError, validate
 
-from rucio.common.config import config_get, config_get_int
+from rucio.common.config import config_get, config_get_int, config_get_list
 from rucio.common.exception import InvalidObject, TraceValidationSchemaNotFound
 from rucio.common.logging import rucio_log_formatter
 from rucio.common.schema.generic import TIME_ENTRY, UUID, IPv4orIPv6
@@ -70,7 +70,7 @@ ROTATING_LOGGER.addHandler(ROTATING_HANDLER)
 
 BROKERS_ALIAS, BROKERS_RESOLVED = [], []
 try:
-    BROKERS_ALIAS = [b.strip() for b in config_get('trace', 'brokers').split(',')]
+    BROKERS_ALIAS = config_get_list('trace', 'brokers')
 except:
     raise Exception('Could not load brokers from configuration')
 

rucio/core/transfer.py

@@ -38,7 +38,6 @@ from rucio.core import request as request_core
 from rucio.core.account import list_accounts
 from rucio.core.monitor import MetricManager
 from rucio.core.request import DirectTransfer, RequestSource, RequestWithSources, TransferDestination, transition_request_state
-from rucio.core.rse import RseData
 from rucio.core.rse_expression_parser import parse_expression
 from rucio.db.sqla import models
 from rucio.db.sqla.constants import DIDType, RequestState, RequestType, TransferLimitDirection
@@ -48,7 +47,6 @@ from rucio.transfertool.bittorrent import BittorrentTransfertool
 from rucio.transfertool.fts3 import FTS3Transfertool
 from rucio.transfertool.globus import GlobusTransferTool
 from rucio.transfertool.mock import MockTransfertool
-from rucio.transfertool.transfertool import TransferStatusReport, Transfertool
 
 if TYPE_CHECKING:
     from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
@@ -56,9 +54,11 @@ if TYPE_CHECKING:
 
     from sqlalchemy.orm import Session
 
-    from rucio.common.types import InternalAccount
+    from rucio.common.types import InternalAccount, LFNDict
+    from rucio.core.rse import RseData
     from rucio.core.topology import Topology
     from rucio.rse.protocols.protocol import RSEProtocol
+    from rucio.transfertool.transfertool import TransferStatusReport, Transfertool
 
     LoggerFunction = Callable[..., Any]
 
@@ -90,7 +90,7 @@ class ProtocolFactory:
     def __init__(self):
         self.protocols = {}
 
-    def protocol(self, rse: RseData, scheme: "Optional[str]", operation: str):
+    def protocol(self, rse: 'RseData', scheme: "Optional[str]", operation: str):
         protocol_key = '%s_%s_%s' % (operation, rse.id, scheme)
         protocol = self.protocols.get(protocol_key)
         if not protocol:
@@ -210,7 +210,12 @@ class DirectTransferImplementation(DirectTransfer):
         # Compute the source URL
         source_sign_url = src.rse.attributes.get(RseAttr.SIGN_URL, None)
         dest_sign_url = dst.rse.attributes.get(RseAttr.SIGN_URL, None)
-        source_url = list(protocol.lfns2pfns(lfns={'scope': rws.scope.external, 'name': rws.name, 'path': src.file_path}).values())[0]
+        lfn: "LFNDict" = {
+            'scope': rws.scope.external,  # type: ignore (scope.external might be None)
+            'name': rws.name,
+            'path': src.file_path
+        }
+        source_url = list(protocol.lfns2pfns(lfns=lfn).values())[0]
         source_url = cls.__rewrite_source_url(source_url, source_sign_url=source_sign_url, dest_sign_url=dest_sign_url, source_scheme=src.scheme)
         return source_url
 
@@ -223,7 +228,11 @@ class DirectTransferImplementation(DirectTransfer):
         protocol = protocol_factory.protocol(dst.rse, dst.scheme, operation)
 
         if dst.rse.info['deterministic']:
-            dest_url = list(protocol.lfns2pfns(lfns={'scope': rws.scope.external, 'name': rws.name}).values())[0]
+            lfn: "LFNDict" = {
+                'scope': rws.scope.external,  # type: ignore (scope.external might be None)
+                'name': rws.name
+            }
+            dest_url = list(protocol.lfns2pfns(lfns=lfn).values())[0]
         else:
             # compute dest url in case of non deterministic
             # naming convention, etc.
@@ -236,7 +245,12 @@ class DirectTransferImplementation(DirectTransfer):
                 if rws.retry_count or rws.activity == 'Recovery':
                     dest_path = '%s_%i' % (dest_path, int(time.time()))
 
-            dest_url = list(protocol.lfns2pfns(lfns={'scope': rws.scope.external, 'name': rws.name, 'path': dest_path}).values())[0]
+            lfn: "LFNDict" = {
+                'scope': rws.scope.external,  # type: ignore (scope.external might be None)
+                'name': rws.name,
+                'path': dest_path
+            }
+            dest_url = list(protocol.lfns2pfns(lfns=lfn).values())[0]
 
         dest_sign_url = dst.rse.attributes.get(RseAttr.SIGN_URL, None)
         dest_url = cls.__rewrite_dest_url(dest_url, dest_sign_url=dest_sign_url)
@@ -498,7 +512,7 @@ def set_transfers_state(
 
 @transactional_session
 def update_transfer_state(
-        tt_status_report: TransferStatusReport,
+        tt_status_report: 'TransferStatusReport',
         stats_manager: request_core.TransferStatsManager,
         *,
         session: "Session",
@@ -744,7 +758,7 @@ def _create_stagein_definitions(
     """
     transfers_by_source = {
         source.rse: [
-            cast(DirectTransfer, StageinTransferImplementation(
+            cast('DirectTransfer', StageinTransferImplementation(
                 source=RequestSource(
                     rse=source.rse,
                     file_path=source.file_path,
@@ -973,7 +987,7 @@ class EnforceSourceRSEExpression(SourceFilterStrategy):
         return self._RankingContext(self, rws, allowed_source_rses)
 
     def apply(self, ctx: RequestRankingContext, source: RequestSource) -> "Optional[int | _SkipSource]":
-        ctx = cast(EnforceSourceRSEExpression._RankingContext, ctx)
+        ctx = cast('EnforceSourceRSEExpression._RankingContext', ctx)
         if ctx.allowed_source_rses is not None and source.rse.id not in ctx.allowed_source_rses:
             return SKIP_SOURCE
 
@@ -1049,7 +1063,7 @@ class PathDistance(SourceRankingStrategy):
         return PathDistance._RankingContext(self, rws, paths_for_rws)
 
     def apply(self, ctx: RequestRankingContext, source: RequestSource) -> "Optional[int | _SkipSource]":
-        path = cast(PathDistance._RankingContext, ctx).paths_for_rws.get(source.rse)
+        path = cast('PathDistance._RankingContext', ctx).paths_for_rws.get(source.rse)
         if not path:
             return SKIP_SOURCE
         return path[0].src.distance
@@ -1057,7 +1071,7 @@ class PathDistance(SourceRankingStrategy):
 
 class PreferSingleHop(PathDistance):
     def apply(self, ctx: RequestRankingContext, source: RequestSource) -> "Optional[int | _SkipSource]":
-        path = cast(PathDistance._RankingContext, ctx).paths_for_rws.get(source.rse)
+        path = cast('PathDistance._RankingContext', ctx).paths_for_rws.get(source.rse)
         if not path:
             return SKIP_SOURCE
         return int(len(path) > 1)
@@ -1097,7 +1111,7 @@ class FailureRate(SourceRankingStrategy):
             self.source_stats.setdefault(stat['src_rse_id'], self._FailureRateStat()).incorporate_stat(stat)
 
     def apply(self, ctx: RequestRankingContext, source: RequestSource) -> "Optional[int | _SkipSource]":
-        failure_rate = cast(FailureRate, ctx.strategy).source_stats.get(source.rse.id, self._FailureRateStat()).get_failure_rate()
+        failure_rate = cast('FailureRate', ctx.strategy).source_stats.get(source.rse.id, self._FailureRateStat()).get_failure_rate()
         return failure_rate
 
 
@@ -1105,7 +1119,7 @@ class SkipSchemeMissmatch(PathDistance):
     filter_only = True
 
     def apply(self, ctx: RequestRankingContext, source: RequestSource) -> "Optional[int | _SkipSource]":
-        path = cast(PathDistance._RankingContext, ctx).paths_for_rws.get(source.rse)
+        path = cast('PathDistance._RankingContext', ctx).paths_for_rws.get(source.rse)
         # path == None means that there is no path;
         # path == [] means that a path exists (according to distances) but cannot be used (scheme mismatch)
         if path is not None and not path:
@@ -1117,7 +1131,7 @@ class SkipIntermediateTape(PathDistance):
 
     def apply(self, ctx: RequestRankingContext, source: RequestSource) -> "Optional[int | _SkipSource]":
         # Discard multihop transfers which contain a tape source as an intermediate hop
-        path = cast(PathDistance._RankingContext, ctx).paths_for_rws.get(source.rse)
+        path = cast('PathDistance._RankingContext', ctx).paths_for_rws.get(source.rse)
         if path and any(transfer.src.rse.is_tape_or_staging_required() for transfer in path[1:]):
             return SKIP_SOURCE
 

rucio/core/volatile_replica.py

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from collections.abc import Iterable
 from datetime import datetime
 from typing import TYPE_CHECKING, Any
 
@@ -26,11 +25,13 @@ from rucio.db.sqla.constants import ReplicaState
 from rucio.db.sqla.session import transactional_session
 
 if TYPE_CHECKING:
+    from collections.abc import Iterable
+
     from sqlalchemy.orm import Session
 
 
 @transactional_session
-def add_volatile_replicas(rse_id: str, replicas: Iterable[dict[str, Any]], *, session: "Session") -> None:
+def add_volatile_replicas(rse_id: str, replicas: "Iterable[dict[str, Any]]", *, session: "Session") -> None:
     """
     Bulk add volatile replicas.
 
@@ -112,7 +113,7 @@ def add_volatile_replicas(rse_id: str, replicas: Iterable[dict[str, Any]], *, se
 
 
 @transactional_session
-def delete_volatile_replicas(rse_id: str, replicas: Iterable[dict[str, Any]], *, session: "Session") -> None:
+def delete_volatile_replicas(rse_id: str, replicas: "Iterable[dict[str, Any]]", *, session: "Session") -> None:
     """
     Bulk delete volatile replicas.
 

rucio/daemons/atropos/atropos.py

@@ -81,7 +81,7 @@ def atropos(
 
 def run_once(
         heartbeat_handler: 'HeartbeatHandler',
-        activity: None,    # NOQA, pylint: disable=W0613
+        activity: None,    # NOQA: ARG001
         date_check: datetime.datetime,
         dry_run: bool,
         grace_period: int,

rucio/daemons/auditor/__init__.py

@@ -16,9 +16,9 @@ import bz2
 import glob
 import logging
 import os
-import queue as Queue
 import select
 from datetime import datetime, timedelta
+from queue import Empty as EmptyQueue
 from typing import TYPE_CHECKING, Optional
 
 from rucio.common import config
@@ -238,7 +238,7 @@ def check(
     while not terminate.is_set():
         try:
             rse, attempts = queue.get(timeout=30)
-        except Queue.Empty:
+        except EmptyQueue:
             continue
         start = datetime.now()
         try:

rucio/daemons/auditor/srmdumps.py

@@ -12,15 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import configparser as ConfigParser
 import datetime
 import glob
 import hashlib
-import html.parser as HTMLParser
 import logging
 import operator
 import os
 import re
+from configparser import RawConfigParser
+from html.parser import HTMLParser
 from typing import IO, TYPE_CHECKING, Any, Optional
 
 import gfal2
@@ -49,7 +49,7 @@ __DUMPERCONFIGDIRS = list(
 OBJECTSTORE_NUM_TRIES = 30
 
 
-class Parser(ConfigParser.RawConfigParser):
+class Parser(RawConfigParser):
     '''
     RawConfigParser subclass that doesn't modify the the name of the options
     and removes any quotes around the string values.
@@ -137,7 +137,7 @@ def gfal_links(base_url: str) -> list[str]:
     return ['/'.join((base_url, f)) for f in ctxt.listdir(str(base_url))]
 
 
-class _LinkCollector(HTMLParser.HTMLParser):
+class _LinkCollector(HTMLParser):
     def __init__(self):
         super(_LinkCollector, self).__init__()
         self.links = []
@@ -244,7 +244,7 @@ def parse_configuration(conf_dirs: Optional[list[str]] = None) -> Parser:
 
 def download_rse_dump(
         rse: str,
-        configuration: ConfigParser.RawConfigParser,
+        configuration: RawConfigParser,
         date: Optional[datetime.datetime] = None,
         destdir: str = DUMPS_CACHE_DIR
 ) -> tuple[str, datetime.datetime]:
@@ -331,7 +331,7 @@ def download_rse_dump(
 
 def generate_url(
         rse: str,
-        config: ConfigParser.RawConfigParser
+        config: RawConfigParser
 ) -> tuple[str, str]:
     '''
     :param rse: Name of the endpoint.

rucio/daemons/automatix/automatix.py

@@ -28,7 +28,7 @@ import rucio.db.sqla.util
 from rucio.client import Client
 from rucio.client.uploadclient import UploadClient
 from rucio.common import exception
-from rucio.common.config import config_get, config_get_bool, config_get_int
+from rucio.common.config import config_get, config_get_bool, config_get_int, config_get_list
 from rucio.common.logging import setup_logging
 from rucio.common.stopwatch import Stopwatch
 from rucio.common.types import InternalScope, LoggerFunction
@@ -149,18 +149,14 @@ def run_once(heartbeat_handler: HeartbeatHandler, inputfile: str, **_kwargs) ->
 
     _, _, logger = heartbeat_handler.live()
     try:
-        rses = [
-            s.strip() for s in config_get("automatix", "rses").split(",")
-        ]  # TODO use config_get_list
+        rses = config_get_list("automatix", "rses")
     except (NoOptionError, NoSectionError, RuntimeError):
         logging.log(
             logging.ERROR,
             "Option rses not found in automatix section. Trying the legacy sites option",
         )
         try:
-            rses = [
-                s.strip() for s in config_get("automatix", "sites").split(",")
-            ]  # TODO use config_get_list
+            rses = config_get_list("automatix", "sites")
             logging.log(
                 logging.WARNING,
                 "Option sites found in automatix section. This option will be deprecated soon. Please update your config to use rses.",
@@ -190,6 +186,8 @@ def run_once(heartbeat_handler: HeartbeatHandler, inputfile: str, **_kwargs) ->
     logger(logging.DEBUG, "Probabilities %s", probabilities)
 
     cycle_stopwatch = Stopwatch()
+    successes = []
+    failures = []
     for rse in rses:
         stopwatch = Stopwatch()
         _, _, logger = heartbeat_handler.live()
@@ -239,24 +237,37 @@ def run_once(heartbeat_handler: HeartbeatHandler, inputfile: str, **_kwargs) ->
                     file_["dataset_meta"]["lifetime"] = dataset_lifetime
             files.append(file_)
         logger(logging.INFO, "Upload %s:%s to %s", scope, dsn, rse)
-        upload_client = UploadClient(client)
-        ret = upload_client.upload(files)
-        if ret == 0:
-            logger(logging.INFO, "%s successfully registered" % dsn)
-            METRICS.counter(name="addnewdataset.done").inc()
-            METRICS.counter(name="addnewfile.done").inc(nbfiles)
-            METRICS.timer(name='datasetinjection').observe(stopwatch.elapsed)
-        else:
-            logger(logging.INFO, "Error uploading files")
-        for physical_fname in physical_fnames:
-            remove(physical_fname)
-        rmdir(tmpdir)
+        try:
+            upload_client = UploadClient(client)
+            ret = upload_client.upload(files)
+            if ret == 0:
+                logger(logging.INFO, "%s successfully registered on %s", dsn, rse)
+                METRICS.counter(name="addnewdataset.done").inc()
+                METRICS.counter(name="addnewfile.done").inc(nbfiles)
+                METRICS.timer(name='datasetinjection').observe(stopwatch.elapsed)
+                successes.append(rse)
+            else:
+                logger(logging.INFO, "Error uploading files")
+                failures.append(rse)
+        except Exception as error:
+            logger(logging.ERROR, "Error uploading files on %s: %s", rse, str(error))
+            failures.append(rse)
+        finally:
+            for physical_fname in physical_fnames:
+                remove(physical_fname)
+            rmdir(tmpdir)
     logger(
         logging.INFO,
-        "It took %f seconds to upload one dataset on %s",
+        "It took %f seconds to upload datasets on %s RSEs: %s",
         cycle_stopwatch.elapsed,
-        str(rses),
+        len(successes),
+        str(successes),
     )
+    if failures:
+        logger(
+            logging.WARNING,
+            "Datasets could not be uploaded on %s RSEs: %s", len(failures), str(failures),
+        )
     return True
 
 

rucio/daemons/badreplicas/necromancer.py

@@ -25,7 +25,7 @@ from sqlalchemy.exc import DatabaseError
 
 import rucio.db.sqla.util
 from rucio.common import exception
-from rucio.common.cache import make_region_memcached
+from rucio.common.cache import MemcacheRegion
 from rucio.common.config import config_get_int
 from rucio.common.exception import DatabaseException
 from rucio.common.logging import setup_logging
@@ -41,7 +41,7 @@ if TYPE_CHECKING:
 
 graceful_stop = threading.Event()
 METRICS = MetricManager(module=__name__)
-REGION = make_region_memcached(expiration_time=config_get_int('necromancer', 'cache_time', False, 600))
+REGION = MemcacheRegion(expiration_time=config_get_int('necromancer', 'cache_time', False, 600))
 DAEMON_NAME = 'necromancer'
 
 

rucio/daemons/bb8/nuclei_background_rebalance.py

@@ -61,7 +61,7 @@ total_total = 0
 global_ratio = float(0)
 for rse in rses:
     site_name = get_rse_attribute(rse['id'], RseAttr.SITE)
-    rse['groupdisk'] = group_space(site_name)
+    rse['groupdisk'] = group_space(site_name)  # type: ignore (site_name could be None)
     rse['primary'] = get_rse_usage(rse_id=rse['id'], source='rucio')[0]['used'] - get_rse_usage(rse_id=rse['id'], source='expired')[0]['used']
     rse['primary'] += rse['groupdisk']
     rse['secondary'] = get_rse_usage(rse_id=rse['id'], source='expired')[0]['used']

rucio/daemons/bb8/t2_background_rebalance.py

@@ -61,7 +61,7 @@ total_total = 0
 global_ratio = float(0)
 for rse in rses:
     site_name = get_rse_attribute(rse['id'], RseAttr.SITE)
-    rse['groupdisk'] = group_space(site_name)
+    rse['groupdisk'] = group_space(site_name)  # type: ignore (site_name could be None)
     rse['primary'] = get_rse_usage(rse_id=rse['id'], source='rucio')[0]['used'] - get_rse_usage(rse_id=rse['id'], source='expired')[0]['used']
     rse['primary'] += rse['groupdisk']
     rse['secondary'] = get_rse_usage(rse_id=rse['id'], source='expired')[0]['used']