rucio 35.7.0__py3-none-any.whl → 37.0.0rc2__py3-none-any.whl

rucio/daemons/reaper/reaper.py

@@ -12,9 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-'''
+"""
 Reaper is a daemon to manage file deletion.
-'''
+"""
 
 import concurrent.futures.thread  # noqa (https://github.com/rucio/rucio/issues/6548)
 
@@ -33,7 +33,7 @@ from dogpile.cache.api import NoValue
 from sqlalchemy.exc import DatabaseError, IntegrityError
 
 import rucio.db.sqla.util
-from rucio.common.cache import make_region_memcached
+from rucio.common.cache import MemcacheRegion
 from rucio.common.config import config_get_bool, config_get_int
 from rucio.common.constants import RseAttr
 from rucio.common.exception import DatabaseException, ReplicaNotFound, ReplicaUnAvailable, ResourceTemporaryUnavailable, RSEAccessDenied, RSENotFound, RSEProtocolNotSupported, ServiceUnavailable, SourceNotFound, VONotFound
@@ -57,12 +57,12 @@ if TYPE_CHECKING:
     from collections.abc import Iterable, Sequence
     from types import FrameType
 
-    from rucio.common.types import LoggerFunction
+    from rucio.common.types import LFNDict, LoggerFunction
     from rucio.daemons.common import HeartbeatHandler
 
 GRACEFUL_STOP = threading.Event()
 METRICS = MetricManager(module=__name__)
-REGION = make_region_memcached(expiration_time=600)
+REGION = MemcacheRegion(expiration_time=600)
 DAEMON_NAME = 'reaper'
 
 EXCLUDED_RSE_GAUGE = METRICS.gauge('excluded_rses.{rse}', documentation='Temporarly excluded RSEs')
@@ -474,7 +474,7 @@ def run_once(
 
     rses_to_process = get_rses_to_process(rses, include_rses, exclude_rses, vos)
     if not rses_to_process:
-        logger(logging.ERROR, 'Reaper: No RSEs found. Will sleep for 30 seconds')
+        logger(logging.WARNING, 'Reaper: No RSEs found, sleeping')
         return must_sleep
     else:
         rses_to_process = [RseData(id_=rse['id'], name=rse['rse'], columns=rse) for rse in rses_to_process]
@@ -632,8 +632,13 @@ def _run_once(
                 del_start_time = time.time()
                 for replica in file_replicas:
                     try:
+                        lfn: "LFNDict" = {
+                            'scope': replica['scope'].external,
+                            'name': replica['name'],
+                            'path': replica['path']
+                        }
                         replica['pfn'] = str(list(rsemgr.lfns2pfns(rse_settings=rse.info,
-                                                                   lfns=[{'scope': replica['scope'].external, 'name': replica['name'], 'path': replica['path']}],
+                                                                   lfns=[lfn],
                                                                    operation='delete', scheme=scheme).values())[0])
                     except (ReplicaUnAvailable, ReplicaNotFound) as error:
                         logger(logging.WARNING, 'Failed get pfn UNAVAILABLE replica %s:%s on %s with error %s', replica['scope'], replica['name'], rse.name, str(error))
@@ -690,7 +695,6 @@ def run(
 
     :param threads:                The total number of workers.
     :param chunk_size:             The size of chunk for deletion.
-    :param threads_per_worker:     Total number of threads created by each worker.
     :param once:                   If True, only runs one iteration of the main loop.
     :param greedy:                 If True, delete right away replicas with tombstone.
     :param rses:                   List of RSEs the reaper should work against.
@@ -711,14 +715,6 @@ def run(
     if rucio.db.sqla.util.is_old_db():
         raise DatabaseException('Database was not updated, daemon won\'t start')
 
-    logging.log(logging.INFO, 'main: starting processes')
-    rses_to_process = get_rses_to_process(rses, include_rses, exclude_rses, vos)
-    if not rses_to_process:
-        logging.log(logging.ERROR, 'Reaper: No RSEs found. Exiting.')
-        return
-
-    logging.log(logging.INFO, 'Reaper: This instance will work on RSEs: %s', ', '.join([rse['rse'] for rse in rses_to_process]))
-
     logging.log(logging.INFO, 'starting reaper threads')
     threads_list = [threading.Thread(target=reaper, kwargs={'once': once,
                                                             'rses': rses,

rucio/daemons/rsedecommissioner/config.py

@@ -75,7 +75,7 @@ def set_status(
     :param rse_id: RSE ID.
     :param status: RSE decommissioning status.
     """
-    config = attr_to_config(get_rse_attribute(rse_id, RseAttr.DECOMMISSION))
+    config = attr_to_config(get_rse_attribute(rse_id, RseAttr.DECOMMISSION))  # type: ignore (get_rse_attribute could return None)
     config['status'] = status
     # add_rse_attribute can handle updating existing entries too
     add_rse_attribute(rse_id, RseAttr.DECOMMISSION, config_to_attr(config))

rucio/daemons/rsedecommissioner/profiles/generic.py

@@ -14,7 +14,6 @@
 
 """Generic decommissioning profiles."""
 import logging
-from collections.abc import Callable, Iterable
 from datetime import datetime, timedelta
 from typing import TYPE_CHECKING, Any
 
@@ -32,6 +31,8 @@ from rucio.db.sqla.constants import ReplicaState
 from .types import DecommissioningProfile, HandlerOutcome
 
 if TYPE_CHECKING:
+    from collections.abc import Iterable
+
     from rucio.common.types import LoggerFunction
 
 
@@ -137,7 +138,7 @@ def _generic_discover(
     rse: dict[str, Any],
     *,
     logger: "LoggerFunction" = logging.log
-) -> Iterable[dict[str, Any]]:
+) -> 'Iterable[dict[str, Any]]':
     """Discoverer function that calls the listing function from core.rule.
 
     :param rse: RSE table entry as a dictionary.
@@ -186,7 +187,7 @@ def _generic_finalize(
 
 def _process_replicas_with_no_locks(
     rse: dict[str, Any],
-    replicas: Iterable[dict[str, Any]],
+    replicas: 'Iterable[dict[str, Any]]',
     limit: int = 0,
     *,
     logger: "LoggerFunction" = logging.log,
@@ -382,7 +383,7 @@ def _call_for_attention(
     rule: dict[str, Any],
     rse: dict[str, Any],
     *,
-    logger: Callable[..., None] = logging.log
+    logger: "LoggerFunction" = logging.log
 ) -> HandlerOutcome:
     return HandlerOutcome.NEED_ATTENTION
 

rucio/daemons/rsedecommissioner/profiles/types.py

@@ -14,12 +14,13 @@
 
 """Types used for profile definitions."""
 import logging
-from collections.abc import Callable, Iterable
 from dataclasses import dataclass
 from enum import Enum
 from typing import TYPE_CHECKING, Any
 
 if TYPE_CHECKING:
+    from collections.abc import Callable, Iterable
+
     from rucio.common.types import LoggerFunction
 
 
@@ -42,10 +43,10 @@ class DecommissioningProfile:
     """
 
     rse: dict[str, Any]
-    initializer: Callable[..., None]
-    discoverer: Callable[..., Iterable[dict[str, Any]]]
-    handlers: list[tuple[Callable[..., bool], Callable[..., HandlerOutcome]]]
-    finalizer: Callable[..., bool]
+    initializer: "Callable[..., None]"
+    discoverer: "Callable[..., Iterable[dict[str, Any]]]"
+    handlers: list[tuple["Callable[..., bool]", "Callable[..., HandlerOutcome]"]]
+    finalizer: "Callable[..., bool]"
 
     def initialize(
         self,
@@ -59,7 +60,7 @@ class DecommissioningProfile:
         self,
         *,
         logger: "LoggerFunction" = logging.log
-    ) -> Iterable[dict[str, Any]]:
+    ) -> 'Iterable[dict[str, Any]]':
         """Call the discoverer."""
         return self.discoverer(self.rse, logger=logger)
 

rucio/daemons/rsedecommissioner/rse_decommissioner.py

@@ -98,7 +98,7 @@ def run_once(
         # Get the decommission attribute (encodes the decommissioning config)
         attr = get_rse_attribute(rse['id'], RseAttr.DECOMMISSION)
         try:
-            config = attr_to_config(attr)
+            config = attr_to_config(attr)  # type: ignore (attr could be None)
         except InvalidStatusName:
             logger(logging.ERROR, 'RSE %s has an invalid decommissioning status',
                    rse['rse'])

rucio/daemons/storage/consistency/actions.py

@@ -35,7 +35,7 @@ from sqlalchemy.orm.exc import FlushError
 
 from rucio.common import exception
 from rucio.common.logging import formatted_logger, setup_logging
-from rucio.common.types import InternalAccount, InternalScope
+from rucio.common.types import InternalAccount, InternalScope, LFNDict
 from rucio.common.utils import daemon_sleep
 from rucio.core.heartbeat import die, live, sanity_check
 from rucio.core.monitor import MetricManager
@@ -443,16 +443,18 @@ def process_dark_files(
                 logger(logging.INFO, 'Processing a dark file:\n RSE %s  Scope: %s  Name: %s'
                        % (rse, scope, name))
                 rse_id = get_rse_id(rse=rse)
-                Intscope = InternalScope(scope=scope, vo=issuer.vo)
-                lfns = [{'scope': scope, 'name': name}]
-
+                internal_scope = InternalScope(scope=scope, vo=issuer.vo)
+                lfn: "LFNDict" = {
+                    'scope': scope,
+                    'name': name,
+                }
                 attributes = get_rse_info(rse=rse)
-                pfns = lfns2pfns(rse_settings=attributes, lfns=lfns, operation='delete')
+                pfns = lfns2pfns(rse_settings=attributes, lfns=[lfn], operation='delete')
                 pfn_key = scope + ':' + name
                 url = pfns[pfn_key]
                 urls = [url]
                 paths = parse_pfns(attributes, urls, operation='delete')
-                replicas = [{'scope': Intscope, 'rse_id': rse_id, 'name': name,
+                replicas = [{'scope': internal_scope, 'rse_id': rse_id, 'name': name,
                              'path': paths[url]['path'] + paths[url]['name']}]
                 add_quarantined_replicas(rse_id, replicas, session=None)
                 deleted_files += 1