rucio 32.8.6__py3-none-any.whl → 35.8.0__py3-none-any.whl

rucio/daemons/badreplicas/minos.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,41 +14,43 @@
 
 import functools
 import logging
+import math
 import re
 import threading
-from collections.abc import Callable
 from datetime import datetime
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any, Optional
 
-import math
 from sqlalchemy.exc import DatabaseError
 
 import rucio.db.sqla.util
 from rucio.common.config import config_get_int
-from rucio.common.exception import UnsupportedOperation, DataIdentifierNotFound, ReplicaNotFound, DatabaseException
+from rucio.common.exception import DatabaseException, DataIdentifierNotFound, ReplicaNotFound, UnsupportedOperation
 from rucio.common.logging import setup_logging
 from rucio.common.utils import chunks
 from rucio.core.did import get_metadata
-from rucio.core.replica import (get_bad_pfns, get_pfn_to_rse, declare_bad_file_replicas,
-                                get_did_from_pfns, update_replicas_states, bulk_add_bad_replicas,
-                                bulk_delete_bad_pfns, get_replicas_state)
+from rucio.core.replica import bulk_add_bad_replicas, bulk_delete_bad_pfns, declare_bad_file_replicas, get_bad_pfns, get_did_from_pfns, get_pfn_to_rse, get_replicas_state, update_replicas_states
 from rucio.core.rse import get_rse_name
 from rucio.daemons.common import run_daemon
-from rucio.db.sqla.constants import BadFilesStatus, BadPFNStatus, ReplicaState
+from rucio.db.sqla.constants import MYSQL_LOCK_WAIT_TIMEOUT_EXCEEDED, ORACLE_DEADLOCK_DETECTED_REGEX, ORACLE_RESOURCE_BUSY_REGEX, BadFilesStatus, BadPFNStatus, ReplicaState
 from rucio.db.sqla.session import get_session
 
 if TYPE_CHECKING:
+    from collections.abc import Iterable, Mapping
     from types import FrameType
-    from typing import Optional
 
+    from rucio.common.types import InternalAccount, LoggerFunction
     from rucio.daemons.common import HeartbeatHandler
-    from rucio.common.types import InternalAccount
 
 graceful_stop = threading.Event()
 DAEMON_NAME = 'minos'
 
 
-def __classify_bad_pfns(pfns: list) -> tuple[dict, dict]:
+def __classify_bad_pfns(
+        pfns: "Iterable[Mapping[str, Any]]"
+) -> tuple[
+    dict[tuple["InternalAccount", str, str], list[str]],
+    dict[tuple["InternalAccount", str, datetime], list[str]]
+]:
     """
     Function that takes a list of PFNs and classify them in 2 dictionaries : bad_replicas and temporary_unvailables
     :param pfns: List of PFNs
@@ -77,7 +78,11 @@ def __classify_bad_pfns(pfns: list) -> tuple[dict, dict]:
     return bad_replicas, temporary_unvailables
 
 
-def __clean_unknown_replicas(pfns: list, vo: str, logger: "Callable") -> dict:
+def __clean_unknown_replicas(
+        pfns: "Iterable[str]",
+        vo: str,
+        logger: "LoggerFunction"
+) -> dict[str, dict[str, list[str]]]:
     """
     Identify from the list of PFNs the one that are unknown and remove them from the bad_pfns table
     :param pfns: List of PFNs
@@ -111,10 +116,16 @@ def __clean_unknown_replicas(pfns: list, vo: str, logger: "Callable") -> dict:
     return dict_rse
 
 
-def __update_temporary_unavailable(chunk: list, reason: str, expires_at: datetime, account: "InternalAccount", logger: "Callable") -> None:
+def __update_temporary_unavailable(
+        chunk: "Iterable[dict[str, Any]]",
+        reason: str,
+        expires_at: datetime,
+        account: "InternalAccount",
+        logger: "LoggerFunction"
+) -> None:
     """
     Update temporary unavailable replicas one by one
-    :param chunk: List of unvailable replicas to update
+    :param chunk: List of unavailable replicas to update
     :param reason: Reason of the temporary unavailable replica
     :param expires_at: Expiration date of the temporary unavailability
     :param account: Account who declared the replica
@@ -209,7 +220,7 @@ def run_once(heartbeat_handler: "HeartbeatHandler", bulk: int, **_kwargs) -> boo
                         bulk_delete_bad_pfns(pfns=chunk, session=session)
                         session.commit()  # pylint: disable=no-member
         except (DatabaseException, DatabaseError) as error:
-            if re.match('.*ORA-00054.*', error.args[0]) or re.match('.*ORA-00060.*', error.args[0]) or 'ERROR 1205 (HY000)' in error.args[0]:
+            if re.match(ORACLE_RESOURCE_BUSY_REGEX, error.args[0]) or re.match(ORACLE_DEADLOCK_DETECTED_REGEX, error.args[0]) or MYSQL_LOCK_WAIT_TIMEOUT_EXCEEDED in error.args[0]:
                 logger(logging.WARNING, 'Lock detected when handling request - skipping: %s', str(error))
             else:
                 logger(logging.ERROR, 'Exception', exc_info=True)
@@ -262,7 +273,7 @@ def run_once(heartbeat_handler: "HeartbeatHandler", bulk: int, **_kwargs) -> boo
                     __update_temporary_unavailable(chunk=chunk, reason=reason, expires_at=expires_at, account=account, logger=logger)
                     session = get_session()
                 except (DatabaseException, DatabaseError) as error:
-                    if re.match('.*ORA-00054.*', error.args[0]) or re.match('.*ORA-00060.*', error.args[0]) or 'ERROR 1205 (HY000)' in error.args[0]:
+                    if re.match(ORACLE_RESOURCE_BUSY_REGEX, error.args[0]) or re.match(ORACLE_DEADLOCK_DETECTED_REGEX, error.args[0]) or MYSQL_LOCK_WAIT_TIMEOUT_EXCEEDED in error.args[0]:
                         logger(logging.WARNING, 'Lock detected when handling request - skipping: %s', str(error))
                     else:
                         logger(logging.ERROR, 'Exception', exc_info=True)
@@ -304,7 +315,7 @@ def run(threads: int = 1, bulk: int = 100, once: bool = False, sleep_time: int =
             thread_list = [thread.join(timeout=3.14) for thread in thread_list if thread and thread.is_alive()]
 
 
-def stop(signum: "Optional[int]" = None, frame: "Optional[FrameType]" = None) -> None:
+def stop(signum: Optional[int] = None, frame: Optional["FrameType"] = None) -> None:
     """
     Graceful exit.
     """

rucio/daemons/badreplicas/minos_temporary_expiration.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,24 +14,23 @@
 
 import functools
 import logging
+import math
 import re
 import threading
 import traceback
 from typing import TYPE_CHECKING
 
-import math
 from sqlalchemy.exc import DatabaseError
 
 import rucio.db.sqla.util
 from rucio.common import exception
-from rucio.common.exception import DataIdentifierNotFound, ReplicaNotFound, DatabaseException
+from rucio.common.exception import DatabaseException, DataIdentifierNotFound, ReplicaNotFound
 from rucio.common.logging import setup_logging
 from rucio.common.utils import chunks
 from rucio.core.did import get_metadata
-from rucio.core.replica import (update_replicas_states, get_replicas_state,
-                                bulk_delete_bad_replicas, list_expired_temporary_unavailable_replicas)
+from rucio.core.replica import bulk_delete_bad_replicas, get_replicas_state, list_expired_temporary_unavailable_replicas, update_replicas_states
 from rucio.daemons.common import run_daemon
-from rucio.db.sqla.constants import BadFilesStatus, ReplicaState
+from rucio.db.sqla.constants import MYSQL_LOCK_WAIT_TIMEOUT_EXCEEDED, ORACLE_DEADLOCK_DETECTED_REGEX, ORACLE_RESOURCE_BUSY_REGEX, BadFilesStatus, ReplicaState
 from rucio.db.sqla.session import get_session
 
 if TYPE_CHECKING:
@@ -129,7 +127,7 @@ def run_once(heartbeat_handler: "HeartbeatHandler", bulk: int, **_kwargs) -> boo
                     session.commit()  # pylint: disable=no-member
             session = get_session()
         except (DatabaseException, DatabaseError) as error:
-            if re.match('.*ORA-00054.*', error.args[0]) or re.match('.*ORA-00060.*', error.args[0]) or 'ERROR 1205 (HY000)' in error.args[0]:
+            if re.match(ORACLE_RESOURCE_BUSY_REGEX, error.args[0]) or re.match(ORACLE_DEADLOCK_DETECTED_REGEX, error.args[0]) or MYSQL_LOCK_WAIT_TIMEOUT_EXCEEDED in error.args[0]:
                 logger(logging.WARNING, 'Lock detected when handling request - skipping: %s', str(error))
             else:
                 logger(logging.ERROR, 'Exception', exc_info=True)

rucio/daemons/badreplicas/necromancer.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -19,7 +18,7 @@ import re
 import threading
 import time
 from datetime import datetime, timedelta
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Optional
 
 from dogpile.cache.api import NO_VALUE
 from sqlalchemy.exc import DatabaseError
@@ -31,17 +30,14 @@ from rucio.common.config import config_get_int
 from rucio.common.exception import DatabaseException
 from rucio.common.logging import setup_logging
 from rucio.core.monitor import MetricManager
-from rucio.core.replica import list_bad_replicas, get_replicas_state, get_bad_replicas_backlog
-from rucio.core.rule import (update_rules_for_lost_replica, update_rules_for_bad_replica,
-                             get_evaluation_backlog)
-from rucio.daemons.common import run_daemon
-from rucio.db.sqla.constants import ReplicaState
+from rucio.core.replica import get_bad_replicas_backlog, get_replicas_state, list_bad_replicas
+from rucio.core.rule import get_evaluation_backlog, update_rules_for_bad_replica, update_rules_for_lost_replica
+from rucio.daemons.common import HeartbeatHandler, run_daemon
+from rucio.db.sqla.constants import MYSQL_LOCK_WAIT_TIMEOUT_EXCEEDED, ORACLE_DEADLOCK_DETECTED_REGEX, ORACLE_RESOURCE_BUSY_REGEX, ReplicaState
 
 if TYPE_CHECKING:
     from types import FrameType
-    from typing import Optional
 
-    from rucio.daemons.common import HeartbeatHandler
 
 graceful_stop = threading.Event()
 METRICS = MetricManager(module=__name__)
@@ -71,7 +67,7 @@ def necromancer(bulk: int, once: bool = False, sleep_time: int = 60) -> None:
     )
 
 
-def run_once(heartbeat_handler: "HeartbeatHandler", bulk: int, **_kwargs) -> bool:
+def run_once(heartbeat_handler: HeartbeatHandler, bulk: int, **_kwargs) -> bool:
     worker_number, total_workers, logger = heartbeat_handler.live()
     must_sleep = True
 
@@ -140,7 +136,7 @@ def run_once(heartbeat_handler: "HeartbeatHandler", bulk: int, **_kwargs) -> boo
                         update_rules_for_lost_replica(scope=scope, name=name, rse_id=rse_id, nowait=True)
                         METRICS.counter(name='badfiles.lostfile').inc()
                     except (DatabaseException, DatabaseError) as error:
-                        if re.match('.*ORA-00054.*', error.args[0]) or re.match('.*ORA-00060.*', error.args[0]) or 'ERROR 1205 (HY000)' in error.args[0]:
+                        if re.match(ORACLE_RESOURCE_BUSY_REGEX, error.args[0]) or re.match(ORACLE_DEADLOCK_DETECTED_REGEX, error.args[0]) or MYSQL_LOCK_WAIT_TIMEOUT_EXCEEDED in error.args[0]:
                             logger(logging.WARNING, 'Lock detected when handling request - skipping: %s', str(error))
                         else:
                             logger(logging.ERROR, str(error))
@@ -153,7 +149,7 @@ def run_once(heartbeat_handler: "HeartbeatHandler", bulk: int, **_kwargs) -> boo
                         update_rules_for_bad_replica(scope=scope, name=name, rse_id=rse_id, nowait=True)
                         METRICS.counter(name='badfiles.recovering').inc()
                     except (DatabaseException, DatabaseError) as error:
-                        if re.match('.*ORA-00054.*', error.args[0]) or re.match('.*ORA-00060.*', error.args[0]) or 'ERROR 1205 (HY000)' in error.args[0]:
+                        if re.match(ORACLE_RESOURCE_BUSY_REGEX, error.args[0]) or re.match(ORACLE_DEADLOCK_DETECTED_REGEX, error.args[0]) or MYSQL_LOCK_WAIT_TIMEOUT_EXCEEDED in error.args[0]:
                             logger(logging.WARNING, 'Lock detected when handling request - skipping: %s', str(error))
                         else:
                             logger(logging.ERROR, str(error))
@@ -193,7 +189,7 @@ def run(threads: int = 1, bulk: int = 100, once: bool = False, sleep_time: int =
             thread_list = [thread.join(timeout=3.14) for thread in thread_list if thread and thread.is_alive()]
 
 
-def stop(signum: "Optional[int]" = None, frame: "Optional[FrameType]" = None) -> None:
+def stop(signum: Optional[int] = None, frame: Optional["FrameType"] = None) -> None:
     """
     Graceful exit.
     """

rucio/daemons/bb8/__init__.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

rucio/daemons/bb8/bb8.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -21,22 +20,20 @@ import functools
 import logging
 import socket
 import threading
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Optional
 
 from rucio.common.config import config_get_float
 from rucio.common.exception import InvalidRSEExpression
 from rucio.common.logging import setup_logging
-from rucio.core.heartbeat import sanity_check, list_payload_counts
+from rucio.core.heartbeat import list_payload_counts, sanity_check
 from rucio.core.rse import get_rse_usage
 from rucio.core.rse_expression_parser import parse_expression
-from rucio.daemons.bb8.common import rebalance_rse, get_active_locks
-from rucio.daemons.common import run_daemon
+from rucio.daemons.bb8.common import get_active_locks, rebalance_rse
+from rucio.daemons.common import HeartbeatHandler, run_daemon
 
 if TYPE_CHECKING:
     from types import FrameType
-    from typing import Optional
 
-    from rucio.daemons.common import HeartbeatHandler
 
 graceful_stop = threading.Event()
 DAEMON_NAME = "rucio-bb8"
@@ -77,7 +74,7 @@ def rule_rebalancer(
 
 
 def run_once(
-    heartbeat_handler: "HeartbeatHandler",
+    heartbeat_handler: HeartbeatHandler,
     rse_expression: str,
     move_subscriptions: bool,
     use_dump: bool,
@@ -313,7 +310,7 @@ def run_once(
     return must_sleep
 
 
-def stop(signum: "Optional[int]" = None, frame: "Optional[FrameType]" = None) -> None:
+def stop(signum: Optional[int] = None, frame: Optional["FrameType"] = None) -> None:
     """
     Graceful exit.
     """
@@ -338,7 +335,7 @@ def run(
     hostname = socket.gethostname()
     sanity_check(executable=DAEMON_NAME, hostname=hostname)
     logging.info("BB8 starting %s threads", str(threads))
-    threads = [
+    thread_list = [
         threading.Thread(
             target=rule_rebalancer,
             kwargs={
@@ -350,7 +347,7 @@ def run(
         )
         for _ in range(0, threads)
     ]
-    [thread.start() for thread in threads]
+    [thread.start() for thread in thread_list]
     # Interruptible joins require a timeout.
-    while threads[0].is_alive():
-        [thread.join(timeout=3.14) for thread in threads]
+    while thread_list[0].is_alive():
+        [thread.join(timeout=3.14) for thread in thread_list]