rucio 37.4.0__py3-none-any.whl → 37.6.0__py3-none-any.whl

rucio/core/rule.py

@@ -191,7 +191,7 @@ def add_rule(
     logger: LoggerFunction = logging.log
 ) -> list[str]:
     """
-    Adds a replication rule for every did in dids
+    Adds a replication rule for every DID in DIDs
 
     :param dids:                       List of data identifiers.
     :param account:                    Account issuing the rule.
@@ -277,7 +277,7 @@ def add_rule(
         notify_value = {'Y': RuleNotification.YES, 'C': RuleNotification.CLOSE, 'P': RuleNotification.PROGRESS}.get(notify or '', RuleNotification.NO)
 
         for elem in dids:
-            # 3. Get the did
+            # 3. Get the DID
             with METRICS.timer('add_rule.get_did'):
                 try:
                     stmt = select(
@@ -292,7 +292,7 @@ def add_rule(
                 except TypeError as error:
                     raise InvalidObject(error.args) from error  # https://pylint.readthedocs.io/en/latest/user_guide/messages/warning/raise-missing-from.html
 
-            # 3.1 If the did is a constituent, relay the rule to the archive
+            # 3.1 If the DID is a constituent, relay the rule to the archive
             if did.did_type == DIDType.FILE and did.constituent:
                 # Check if a single replica of this DID exists; Do not put rule on file if there are only replicas on TAPE
                 stmt = select(
@@ -407,7 +407,7 @@ def add_rule(
                 logger(logging.DEBUG, "Forced injection of rule %s", str(new_rule.id))
 
             if asynchronous or delay_injection:
-                # TODO: asynchronous mode only available for closed dids (on the whole tree?)
+                # TODO: asynchronous mode only available for closed DIDs (on the whole tree?)
                 new_rule.state = RuleState.INJECT
                 logger(logging.DEBUG, "Created rule %s for injection", str(new_rule.id))
                 if delay_injection:
@@ -485,7 +485,7 @@ def add_rules(
     logger: LoggerFunction = logging.log
 ) -> dict[tuple[InternalScope, str], list[str]]:
     """
-    Adds a list of replication rules to every did in dids
+    Adds a list of replication rules to every DID in DIDs
 
     :params dids:    List of data identifiers.
     :param rules:    List of dictionaries defining replication rules.
@@ -521,7 +521,7 @@ def add_rules(
             all_source_rses = list(set([rse['id'] for rse in all_source_rses]))
 
         for elem in dids:
-            # 2. Get the did
+            # 2. Get the DID
             with METRICS.timer('add_rules.get_did'):
                 try:
                     stmt = select(
@@ -536,7 +536,7 @@ def add_rules(
                 except TypeError as error:
                     raise InvalidObject(error.args) from error
 
-            # 2.1 If the did is a constituent, relay the rule to the archive
+            # 2.1 If the DID is a constituent, relay the rule to the archive
             if did.did_type == DIDType.FILE and did.constituent:  # Check if a single replica of this DID exists
                 stmt = select(
                     func.count()
@@ -584,7 +584,7 @@ def add_rules(
 
             rule_ids[(elem['scope'], elem['name'])] = []
 
-            # 3. Resolve the did into its contents
+            # 3. Resolve the DID into its contents
             with METRICS.timer('add_rules.resolve_dids_to_locks_replicas'):
                 # Get all Replicas, not only the ones interesting for the rse_expression
                 datasetfiles, locks, replicas, source_replicas = __resolve_did_to_locks_and_replicas(did=did,
@@ -828,7 +828,7 @@ def inject_rule(
         dids = [{'scope': dataset['scope'], 'name': dataset['name']} for dataset in rucio.core.did.list_child_datasets(scope=rule.scope, name=rule.name, session=session)]
         # Remove duplicates from the list of dictionaries
         dids = [dict(t) for t in {tuple(d.items()) for d in dids}]
-        # Remove dids which already have a similar rule
+        # Remove DIDs which already have a similar rule
         stmt = select(
             models.ReplicationRule.id
         ).where(
@@ -881,7 +881,7 @@ def inject_rule(
     with METRICS.timer('inject_rule.create_rse_selector'):
         rseselector = RSESelector(account=rule['account'], rses=rses, weight=rule.weight, copies=rule.copies, ignore_account_limit=rule.ignore_account_limit, session=session)
 
-    # 3. Get the did
+    # 3. Get the DID
     with METRICS.timer('inject_rule.get_did'):
         try:
             stmt = select(
@@ -1104,7 +1104,7 @@ def list_associated_rules_for_file(
     :param session: The database session in use.
     :raises:        RucioException
     """
-    rucio.core.did.get_did(scope=scope, name=name, session=session)  # Check if the did actually exists
+    rucio.core.did.get_did(scope=scope, name=name, session=session)  # Check if the DID actually exists
     stmt = select(
         models.ReplicationRule,
         models.DataIdentifier.bytes
@@ -1364,7 +1364,7 @@ def repair_rule(
                 rule.locks_stuck_cnt = count.state_counter
         logger(logging.DEBUG, "Finished resetting counters for rule %s [%d/%d/%d]", str(rule.id), rule.locks_ok_cnt, rule.locks_replicating_cnt, rule.locks_stuck_cnt)
 
-        # Get the did
+        # Get the DID
         stmt = select(
             models.DataIdentifier
         ).where(
@@ -1387,7 +1387,7 @@ def repair_rule(
             hard_repair = True
             logger(logging.DEBUG, 'Repairing rule %s in HARD mode', str(rule.id))
 
-        # Resolve the did to its contents
+        # Resolve the DID to its contents
         datasetfiles, locks, replicas, source_replicas = __resolve_did_to_locks_and_replicas(did=did,
                                                                                              nowait=True,
                                                                                              restrict_rses=[rse['id'] for rse in rses],
@@ -2060,10 +2060,10 @@ def re_evaluate_did(
     logger: LoggerFunction = logging.log
 ) -> None:
     """
-    Re-Evaluates a did.
+    Re-Evaluates a DID.
 
-    :param scope:                   The scope of the did to be re-evaluated.
-    :param name:                    The name of the did to be re-evaluated.
+    :param scope:                   The scope of the DID to be re-evaluated.
+    :param name:                    The name of the DID to be re-evaluated.
     :param rule_evaluation_action:  The Rule evaluation action.
     :param session:                 The database session in use.
     :param logger:                  Optional decorated logger that can be passed from the calling daemons or servers.
@@ -2086,7 +2086,7 @@ def re_evaluate_did(
     else:
         __evaluate_did_detach(did, session=session, logger=logger)
 
-    # Update size and length of did
+    # Update size and length of DID
     if session.bind.dialect.name == 'oracle':
         stmt = select(
             func.sum(models.DataIdentifierAssociation.bytes),
@@ -2118,12 +2118,12 @@ def get_updated_dids(
     session: "Session"
 ) -> list[tuple[str, InternalScope, str, DIDReEvaluation]]:
     """
-    Get updated dids.
+    Get updated DIDs.
 
     :param total_workers:      Number of total workers.
     :param worker_number:      id of the executing worker.
-    :param limit:              Maximum number of dids to return.
-    :param blocked_dids:       Blocked dids to filter.
+    :param limit:              Maximum number of DIDs to return.
+    :param blocked_dids:       Blocked DIDs to filter.
     :param session:            Database session in use.
     """
     blocked_dids = blocked_dids or []
@@ -2135,7 +2135,7 @@ def get_updated_dids(
     )
     stmt = filter_thread_work(session=session, query=stmt, total_threads=total_workers, thread_id=worker_number, hash_variable='name')
 
-    # Remove blocked dids from query, but only do the first 30 ones, not to overload the query
+    # Remove blocked DIDs from query, but only do the first 30 ones, not to overload the query
     if blocked_dids:
         chunk = list(chunks(blocked_dids, 30))[0]
         stmt = stmt.where(tuple_(models.UpdatedDID.scope, models.UpdatedDID.name).notin_(chunk))
@@ -3403,9 +3403,9 @@ def __evaluate_did_detach(
     logger: LoggerFunction = logging.log
 ) -> None:
     """
-    Evaluate a parent did which has children removed.
+    Evaluate a parent DID which has children removed.
 
-    :param eval_did:  The did object in use.
+    :param eval_did:  The DID object in use.
     :param session:   The database session in use.
     :param logger:    Optional decorated logger that can be passed from the calling daemons or servers.
     """
@@ -3563,9 +3563,9 @@ def __evaluate_did_attach(
     logger: LoggerFunction = logging.log
 ) -> None:
     """
-    Evaluate a parent did which has new children
+    Evaluate a parent DID which has new children
 
-    :param eval_did:  The did object in use.
+    :param eval_did:  The DID object in use.
     :param session:   The database session in use.
     :param logger:    Optional decorated logger that can be passed from the calling daemons or servers.
     :raises:          ReplicationRuleCreationTemporaryFailed
@@ -3793,7 +3793,7 @@ def __evaluate_did_attach(
                         # Insert rule history
                         insert_rule_history(rule=rule, recent=True, longterm=False, session=session)
 
-            # Unflage the dids
+            # Unflag the DIDs
             with METRICS.timer('evaluate_did_attach.update_did'):
                 for did in new_child_dids:
                     did.rule_evaluation = None
@@ -3815,9 +3815,9 @@ def __resolve_did_to_locks_and_replicas(
            dict[tuple[str, str], models.RSEFileAssociation],
            dict[tuple[str, str], str]]:
     """
-    Resolves a did to its constituent children and reads the locks and replicas of all the constituent files.
+    Resolves a DID to its constituent children and reads the locks and replicas of all the constituent files.
 
-    :param did:            The db object of the did the rule is applied on.
+    :param did:            The db object of the DID the rule is applied on.
     :param nowait:         Nowait parameter for the FOR UPDATE statement.
     :param restrict_rses:  Possible rses of the rule, so only these replica/locks should be considered.
     :param source_rses:    Source rses for this rule. These replicas are not row-locked.
@@ -3922,7 +3922,7 @@ def __resolve_dids_to_locks_and_replicas(
            dict[tuple[str, str], models.RSEFileAssociation],
            dict[tuple[str, str], str]]:
     """
-    Resolves a list of dids to its constituent children and reads the locks and replicas of all the constituent files.
+    Resolves a list of DIDs to its constituent children and reads the locks and replicas of all the constituent files.
 
     :param dids:           The list of DataIdentifierAssociation objects.
     :param nowait:         Nowait parameter for the FOR UPDATE statement.
@@ -3940,7 +3940,7 @@ def __resolve_dids_to_locks_and_replicas(
     restrict_rses = restrict_rses or []
 
     if dids[0].child_type == DIDType.FILE:
-        # All the dids will be files!
+        # All the DIDs will be files!
         # Prepare the datasetfiles
         files = []
         for did in dids:

rucio/core/rule_grouping.py

@@ -1317,9 +1317,9 @@ def __set_replica_unavailable(replica, *, session: "Session"):
 @transactional_session
 def apply_rule(did, rule, rses, source_rses, rseselector, *, session: "Session", logger=logging.log):
     """
-    Apply a replication rule to one did.
+    Apply a replication rule to one DID.
 
-    :param did:          the did object
+    :param did:          the DID object
     :param rule:         the rule object
     :param rses:         target rses_ids
     :param source_rses:  source rses_ids
@@ -1616,4 +1616,3 @@ def apply_rule(did, rule, rses, source_rses, rseselector, *, session: "Session",
     for rse_id in account_counters_files:
         account_counter.increase(rse_id=rse_id, account=rule.account, files=account_counters_files[rse_id], bytes_=account_counters_bytes[rse_id], session=session)
     session.flush()
-

rucio/core/scope.py

@@ -65,7 +65,7 @@ def add_scope(scope, account, *, session: "Session"):
             raise Duplicate('Scope \'%s\' already exists!' % scope)
         else:
             raise RucioException(e)
-    except:
+    except Exception:
         raise RucioException(str(format_exc()))
 
 

rucio/core/trace.py

@@ -71,7 +71,7 @@ ROTATING_LOGGER.addHandler(ROTATING_HANDLER)
 BROKERS_ALIAS, BROKERS_RESOLVED = [], []
 try:
     BROKERS_ALIAS = config_get_list('trace', 'brokers')
-except:
+except Exception:
     raise Exception('Could not load brokers from configuration')
 
 PORT = config_get_int('trace', 'port')
@@ -285,7 +285,7 @@ for broker in BROKERS_ALIAS:
     try:
         addrinfos = socket.getaddrinfo(broker, 0, socket.AF_INET, 0, socket.IPPROTO_TCP)
         BROKERS_RESOLVED = [ai[4][0] for ai in addrinfos]
-    except:
+    except Exception:
         pass
 
 CONNS = []

rucio/core/transfer.py

@@ -27,11 +27,10 @@ from dogpile.cache.api import NoValue
 from sqlalchemy import select, update
 from sqlalchemy.exc import IntegrityError
 
-from rucio.common import constants
 from rucio.common.config import config_get, config_get_list
-from rucio.common.constants import SUPPORTED_PROTOCOLS, RseAttr
+from rucio.common.constants import SUPPORTED_PROTOCOLS, RseAttr, TransferLimitDirection
 from rucio.common.exception import InvalidRSEExpression, RequestNotFound, RSEProtocolNotSupported, RucioException, UnsupportedOperation
-from rucio.common.utils import construct_non_deterministic_pfn
+from rucio.common.utils import construct_non_deterministic_pfn, get_transfer_schemas
 from rucio.core import did
 from rucio.core import message as message_core
 from rucio.core import request as request_core
@@ -40,7 +39,7 @@ from rucio.core.monitor import MetricManager
 from rucio.core.request import DirectTransfer, RequestSource, RequestWithSources, TransferDestination, transition_request_state
 from rucio.core.rse_expression_parser import parse_expression
 from rucio.db.sqla import models
-from rucio.db.sqla.constants import DIDType, RequestState, RequestType, TransferLimitDirection
+from rucio.db.sqla.constants import DIDType, RequestState, RequestType
 from rucio.db.sqla.session import read_session, stream_session, transactional_session
 from rucio.rse import rsemanager as rsemgr
 from rucio.transfertool.bittorrent import BittorrentTransfertool
@@ -1331,7 +1330,7 @@ def __add_compatible_schemes(schemes, allowed_schemes):
     for scheme in schemes:
         if scheme in allowed_schemes:
             return_schemes.append(scheme)
-            for scheme_map_scheme in constants.SCHEME_MAP.get(scheme, []):
+            for scheme_map_scheme in get_transfer_schemas().get(scheme, []):
                 if scheme_map_scheme not in allowed_schemes:
                     continue
                 else:

rucio/daemons/auditor/__init__.py

@@ -247,7 +247,7 @@ def check(
                                  results_dir)
             if output:
                 process_output(output)
-        except:
+        except Exception:
             elapsed = (datetime.now() - start).total_seconds() / 60
             logger.error('Check of "%s" failed in %d minutes, %d remaining attempts', rse, elapsed, attempts, exc_info=True)
             success = False

rucio/daemons/badreplicas/minos.py

@@ -31,7 +31,7 @@ from rucio.core.did import get_metadata
 from rucio.core.replica import bulk_add_bad_replicas, bulk_delete_bad_pfns, declare_bad_file_replicas, get_bad_pfns, get_did_from_pfns, get_pfn_to_rse, get_replicas_state, update_replicas_states
 from rucio.core.rse import get_rse_name
 from rucio.daemons.common import run_daemon
-from rucio.db.sqla.constants import MYSQL_LOCK_WAIT_TIMEOUT_EXCEEDED, ORACLE_DEADLOCK_DETECTED_REGEX, ORACLE_RESOURCE_BUSY_REGEX, BadFilesStatus, BadPFNStatus, ReplicaState
+from rucio.db.sqla.constants import MYSQL_LOCK_WAIT_TIMEOUT_EXCEEDED, ORACLE_DEADLOCK_DETECTED_REGEX, ORACLE_RESOURCE_BUSY_REGEX, PSQL_PSYCOPG_LOCK_NOT_AVAILABLE_REGEX, BadFilesStatus, BadPFNStatus, ReplicaState
 from rucio.db.sqla.session import get_session
 
 if TYPE_CHECKING:
@@ -220,7 +220,10 @@ def run_once(heartbeat_handler: "HeartbeatHandler", bulk: int, **_kwargs) -> boo
                         bulk_delete_bad_pfns(pfns=chunk, session=session)
                         session.commit()  # pylint: disable=no-member
         except (DatabaseException, DatabaseError) as error:
-            if re.match(ORACLE_RESOURCE_BUSY_REGEX, error.args[0]) or re.match(ORACLE_DEADLOCK_DETECTED_REGEX, error.args[0]) or MYSQL_LOCK_WAIT_TIMEOUT_EXCEEDED in error.args[0]:
+            if (re.match(ORACLE_RESOURCE_BUSY_REGEX, error.args[0])
+                    or re.match(ORACLE_DEADLOCK_DETECTED_REGEX, error.args[0])
+                    or re.match(PSQL_PSYCOPG_LOCK_NOT_AVAILABLE_REGEX, str(error.args[0]))
+                    or MYSQL_LOCK_WAIT_TIMEOUT_EXCEEDED in error.args[0]):
                 logger(logging.WARNING, 'Lock detected when handling request - skipping: %s', str(error))
             else:
                 logger(logging.ERROR, 'Exception', exc_info=True)
@@ -273,7 +276,10 @@ def run_once(heartbeat_handler: "HeartbeatHandler", bulk: int, **_kwargs) -> boo
                     __update_temporary_unavailable(chunk=chunk, reason=reason, expires_at=expires_at, account=account, logger=logger)
                     session = get_session()
                 except (DatabaseException, DatabaseError) as error:
-                    if re.match(ORACLE_RESOURCE_BUSY_REGEX, error.args[0]) or re.match(ORACLE_DEADLOCK_DETECTED_REGEX, error.args[0]) or MYSQL_LOCK_WAIT_TIMEOUT_EXCEEDED in error.args[0]:
+                    if (re.match(ORACLE_RESOURCE_BUSY_REGEX, error.args[0])
+                            or re.match(ORACLE_DEADLOCK_DETECTED_REGEX, error.args[0])
+                            or re.match(PSQL_PSYCOPG_LOCK_NOT_AVAILABLE_REGEX, str(error.args[0]))
+                            or MYSQL_LOCK_WAIT_TIMEOUT_EXCEEDED in error.args[0]):
                         logger(logging.WARNING, 'Lock detected when handling request - skipping: %s', str(error))
                     else:
                         logger(logging.ERROR, 'Exception', exc_info=True)

rucio/daemons/badreplicas/minos_temporary_expiration.py

@@ -30,7 +30,7 @@ from rucio.common.utils import chunks
 from rucio.core.did import get_metadata
 from rucio.core.replica import bulk_delete_bad_replicas, get_replicas_state, list_expired_temporary_unavailable_replicas, update_replicas_states
 from rucio.daemons.common import run_daemon
-from rucio.db.sqla.constants import MYSQL_LOCK_WAIT_TIMEOUT_EXCEEDED, ORACLE_DEADLOCK_DETECTED_REGEX, ORACLE_RESOURCE_BUSY_REGEX, BadFilesStatus, ReplicaState
+from rucio.db.sqla.constants import MYSQL_LOCK_WAIT_TIMEOUT_EXCEEDED, ORACLE_DEADLOCK_DETECTED_REGEX, ORACLE_RESOURCE_BUSY_REGEX, PSQL_PSYCOPG_LOCK_NOT_AVAILABLE_REGEX, BadFilesStatus, ReplicaState
 from rucio.db.sqla.session import get_session
 
 if TYPE_CHECKING:
@@ -127,7 +127,10 @@ def run_once(heartbeat_handler: "HeartbeatHandler", bulk: int, **_kwargs) -> boo
                     session.commit()  # pylint: disable=no-member
             session = get_session()
         except (DatabaseException, DatabaseError) as error:
-            if re.match(ORACLE_RESOURCE_BUSY_REGEX, error.args[0]) or re.match(ORACLE_DEADLOCK_DETECTED_REGEX, error.args[0]) or MYSQL_LOCK_WAIT_TIMEOUT_EXCEEDED in error.args[0]:
+            if (re.match(ORACLE_RESOURCE_BUSY_REGEX, error.args[0])
+                    or re.match(ORACLE_DEADLOCK_DETECTED_REGEX, error.args[0])
+                    or re.match(PSQL_PSYCOPG_LOCK_NOT_AVAILABLE_REGEX, str(error.args[0]))
+                    or MYSQL_LOCK_WAIT_TIMEOUT_EXCEEDED in error.args[0]):
                 logger(logging.WARNING, 'Lock detected when handling request - skipping: %s', str(error))
             else:
                 logger(logging.ERROR, 'Exception', exc_info=True)

rucio/daemons/badreplicas/necromancer.py

@@ -33,7 +33,7 @@ from rucio.core.monitor import MetricManager
 from rucio.core.replica import get_bad_replicas_backlog, get_replicas_state, list_bad_replicas
 from rucio.core.rule import get_evaluation_backlog, update_rules_for_bad_replica, update_rules_for_lost_replica
 from rucio.daemons.common import HeartbeatHandler, run_daemon
-from rucio.db.sqla.constants import MYSQL_LOCK_WAIT_TIMEOUT_EXCEEDED, ORACLE_DEADLOCK_DETECTED_REGEX, ORACLE_RESOURCE_BUSY_REGEX, ReplicaState
+from rucio.db.sqla.constants import MYSQL_LOCK_WAIT_TIMEOUT_EXCEEDED, ORACLE_DEADLOCK_DETECTED_REGEX, ORACLE_RESOURCE_BUSY_REGEX, PSQL_PSYCOPG_LOCK_NOT_AVAILABLE_REGEX, ReplicaState
 
 if TYPE_CHECKING:
     from types import FrameType
@@ -136,7 +136,10 @@ def run_once(heartbeat_handler: HeartbeatHandler, bulk: int, **_kwargs) -> bool:
                         update_rules_for_lost_replica(scope=scope, name=name, rse_id=rse_id, nowait=True)
                         METRICS.counter(name='badfiles.lostfile').inc()
                     except (DatabaseException, DatabaseError) as error:
-                        if re.match(ORACLE_RESOURCE_BUSY_REGEX, error.args[0]) or re.match(ORACLE_DEADLOCK_DETECTED_REGEX, error.args[0]) or MYSQL_LOCK_WAIT_TIMEOUT_EXCEEDED in error.args[0]:
+                        if (re.match(ORACLE_RESOURCE_BUSY_REGEX, error.args[0])
+                                or re.match(ORACLE_DEADLOCK_DETECTED_REGEX, error.args[0])
+                                or re.match(PSQL_PSYCOPG_LOCK_NOT_AVAILABLE_REGEX, str(error.args[0]))
+                                or MYSQL_LOCK_WAIT_TIMEOUT_EXCEEDED in error.args[0]):
                             logger(logging.WARNING, 'Lock detected when handling request - skipping: %s', str(error))
                         else:
                             logger(logging.ERROR, str(error))
@@ -149,7 +152,10 @@ def run_once(heartbeat_handler: HeartbeatHandler, bulk: int, **_kwargs) -> bool:
                         update_rules_for_bad_replica(scope=scope, name=name, rse_id=rse_id, nowait=True)
                         METRICS.counter(name='badfiles.recovering').inc()
                     except (DatabaseException, DatabaseError) as error:
-                        if re.match(ORACLE_RESOURCE_BUSY_REGEX, error.args[0]) or re.match(ORACLE_DEADLOCK_DETECTED_REGEX, error.args[0]) or MYSQL_LOCK_WAIT_TIMEOUT_EXCEEDED in error.args[0]:
+                        if (re.match(ORACLE_RESOURCE_BUSY_REGEX, error.args[0])
+                                or re.match(ORACLE_DEADLOCK_DETECTED_REGEX, error.args[0])
+                                or re.match(PSQL_PSYCOPG_LOCK_NOT_AVAILABLE_REGEX, str(error.args[0]))
+                                or MYSQL_LOCK_WAIT_TIMEOUT_EXCEEDED in error.args[0]):
                             logger(logging.WARNING, 'Lock detected when handling request - skipping: %s', str(error))
                         else:
                             logger(logging.ERROR, str(error))

rucio/daemons/bb8/common.py

@@ -279,7 +279,7 @@ def list_rebalance_rule_candidates(
 
     vo = get_rse_vo(rse_id=rse_id)
 
-    # dumps can be applied only for decommission since the dumps doesn't contain info from dids
+    # dumps can be applied only for decommission since the dumps doesn't contain info from DIDs
     if mode == "decommission":
         return _list_rebalance_rule_candidates_dump(rse_id, mode)
 

rucio/daemons/bb8/nuclei_background_rebalance.py

@@ -43,7 +43,7 @@ def group_space(site: str) -> int:
     group_total = 0
     try:
         site_groupdisks = parse_expression('site=%s&spacetoken=ATLASDATADISK&type=GROUPDISK' % site)
-    except:
+    except Exception:
         return group_total
 
     for rse in site_groupdisks:

rucio/daemons/bb8/t2_background_rebalance.py

@@ -43,7 +43,7 @@ def group_space(site: str) -> int:
     group_total = 0
     try:
         site_groupdisks = parse_expression('site=%s&spacetoken=ATLASDATADISK&type=GROUPDISK' % site)
-    except:
+    except Exception:
         return group_total
 
     for rse in site_groupdisks:

rucio/daemons/cache/consumer.py

@@ -103,7 +103,7 @@ class AMQConsumer:
             else:
                 self.__logger(logging.DEBUG, 'Check failed: %s %s '
                               % (isinstance(msg, dict), 'operation' in msg.keys()))
-        except:
+        except Exception:
             self.__logger(logging.ERROR, str(format_exc()))
 
 

rucio/daemons/conveyor/finisher.py

@@ -43,7 +43,7 @@ from rucio.core.rse import list_rses
 from rucio.core.topology import ExpiringObjectCache, Topology
 from rucio.core.transfer import ProtocolFactory
 from rucio.daemons.common import ProducerConsumerDaemon, db_workqueue
-from rucio.db.sqla.constants import MYSQL_LOCK_WAIT_TIMEOUT_EXCEEDED, ORACLE_DEADLOCK_DETECTED_REGEX, ORACLE_RESOURCE_BUSY_REGEX, BadFilesStatus, ReplicaState, RequestState, RequestType
+from rucio.db.sqla.constants import MYSQL_LOCK_WAIT_TIMEOUT_EXCEEDED, ORACLE_DEADLOCK_DETECTED_REGEX, ORACLE_RESOURCE_BUSY_REGEX, PSQL_PSYCOPG_LOCK_NOT_AVAILABLE_REGEX, BadFilesStatus, ReplicaState, RequestState, RequestType
 from rucio.db.sqla.session import transactional_session
 
 if TYPE_CHECKING:
@@ -140,7 +140,10 @@ def _handle_requests(
         logger(logging.DEBUG, 'Finish to update %s finished requests in %s seconds', len(reqs), total_stopwatch.elapsed)
 
     except (DatabaseException, DatabaseError) as error:
-        if re.match(ORACLE_RESOURCE_BUSY_REGEX, error.args[0]) or re.match(ORACLE_DEADLOCK_DETECTED_REGEX, error.args[0]) or MYSQL_LOCK_WAIT_TIMEOUT_EXCEEDED in error.args[0]:
+        if (re.match(ORACLE_RESOURCE_BUSY_REGEX, error.args[0])
+                or re.match(ORACLE_DEADLOCK_DETECTED_REGEX, error.args[0])
+                or re.match(PSQL_PSYCOPG_LOCK_NOT_AVAILABLE_REGEX, str(error.args[0]))
+                or MYSQL_LOCK_WAIT_TIMEOUT_EXCEEDED in error.args[0]):
             logger(logging.WARNING, 'Lock detected when handling request - skipping: %s', str(error))
         else:
             raise
@@ -421,14 +424,20 @@ def __handle_terminated_replicas(
                     try:
                         __update_replica(replica, logger=logger)
                     except (DatabaseException, DatabaseError) as error:
-                        if re.match(ORACLE_RESOURCE_BUSY_REGEX, error.args[0]) or re.match(ORACLE_DEADLOCK_DETECTED_REGEX, error.args[0]) or MYSQL_LOCK_WAIT_TIMEOUT_EXCEEDED in error.args[0]:
+                        if (re.match(ORACLE_RESOURCE_BUSY_REGEX, error.args[0])
+                                or re.match(ORACLE_DEADLOCK_DETECTED_REGEX, error.args[0])
+                                or re.match(PSQL_PSYCOPG_LOCK_NOT_AVAILABLE_REGEX, str(error.args[0]))
+                                or MYSQL_LOCK_WAIT_TIMEOUT_EXCEEDED in error.args[0]):
                             logger(logging.WARNING, "Locks detected when handling replica %s:%s at RSE %s", replica['scope'], replica['name'], replica['rse_id'])
                         else:
                             logger(logging.ERROR, "Could not finish handling replicas %s:%s at RSE %s", replica['scope'], replica['name'], replica['rse_id'], exc_info=True)
                     except Exception as error:
                         logger(logging.ERROR, "Something unexpected happened when updating replica state for transfer %s:%s at %s (%s)", replica['scope'], replica['name'], replica['rse_id'], str(error))
             except (DatabaseException, DatabaseError) as error:
-                if re.match(ORACLE_RESOURCE_BUSY_REGEX, error.args[0]) or re.match(ORACLE_DEADLOCK_DETECTED_REGEX, error.args[0]) or MYSQL_LOCK_WAIT_TIMEOUT_EXCEEDED in error.args[0]:
+                if (re.match(ORACLE_RESOURCE_BUSY_REGEX, error.args[0])
+                        or re.match(ORACLE_DEADLOCK_DETECTED_REGEX, error.args[0])
+                        or re.match(PSQL_PSYCOPG_LOCK_NOT_AVAILABLE_REGEX, str(error.args[0]))
+                        or MYSQL_LOCK_WAIT_TIMEOUT_EXCEEDED in error.args[0]):
                     logger(logging.WARNING, "Locks detected when handling replicas on %s rule %s, update updated time.", req_type, rule_id)
                     try:
                         request_core.touch_requests_by_rule(rule_id)

rucio/daemons/conveyor/poller.py

@@ -40,7 +40,7 @@ from rucio.core import transfer as transfer_core
 from rucio.core.monitor import MetricManager
 from rucio.core.topology import ExpiringObjectCache, Topology
 from rucio.daemons.common import ProducerConsumerDaemon, db_workqueue
-from rucio.db.sqla.constants import MYSQL_LOCK_WAIT_TIMEOUT_EXCEEDED, ORACLE_DEADLOCK_DETECTED_REGEX, ORACLE_RESOURCE_BUSY_REGEX, RequestState, RequestType
+from rucio.db.sqla.constants import MYSQL_LOCK_WAIT_TIMEOUT_EXCEEDED, ORACLE_DEADLOCK_DETECTED_REGEX, ORACLE_RESOURCE_BUSY_REGEX, PSQL_PSYCOPG_LOCK_NOT_AVAILABLE_REGEX, RequestState, RequestType
 from rucio.transfertool.fts3 import FTS3Transfertool
 
 if TYPE_CHECKING:
@@ -387,7 +387,10 @@ def _poll_transfers(
             # Otherwise if one bulk transfer includes many requests and one is not terminated, the transfer will be poll again.
             transfer_core.touch_transfer(transfertool_obj.external_host, transfer_id)
         except (DatabaseException, DatabaseError) as error:
-            if re.match(ORACLE_RESOURCE_BUSY_REGEX, error.args[0]) or re.match(ORACLE_DEADLOCK_DETECTED_REGEX, error.args[0]) or MYSQL_LOCK_WAIT_TIMEOUT_EXCEEDED in error.args[0]:
+            if (re.match(ORACLE_RESOURCE_BUSY_REGEX, error.args[0])
+                    or re.match(ORACLE_DEADLOCK_DETECTED_REGEX, error.args[0])
+                    or re.match(PSQL_PSYCOPG_LOCK_NOT_AVAILABLE_REGEX, str(error.args[0]))
+                    or MYSQL_LOCK_WAIT_TIMEOUT_EXCEEDED in error.args[0]):
                 logger(logging.WARNING, "Lock detected when handling request %s - skipping" % transfer_id)
             else:
                 logger(logging.ERROR, 'Exception', exc_info=True)

rucio/daemons/conveyor/receiver.py

@@ -151,7 +151,7 @@ def receiver(
     use_ssl = True
     try:
         use_ssl = config_get_bool('messaging-fts3', 'use_ssl')
-    except:
+    except Exception:
         logging.info('could not find use_ssl in configuration -- please update your rucio.cfg')
 
     port = config_get_int('messaging-fts3', 'port')

rucio/daemons/conveyor/throttler.py

@@ -26,13 +26,14 @@ from sqlalchemy import null
 
 import rucio.db.sqla.util
 from rucio.common import exception
+from rucio.common.constants import TransferLimitDirection
 from rucio.common.logging import setup_logging
 from rucio.core.monitor import MetricManager
 from rucio.core.request import get_request_stats, re_sync_all_transfer_limits, release_all_waiting_requests, release_waiting_requests_fifo, release_waiting_requests_grouped_fifo, reset_stale_waiting_requests, set_transfer_limit_stats
 from rucio.core.rse import RseCollection, RseData
 from rucio.core.transfer import applicable_rse_transfer_limits
 from rucio.daemons.common import ProducerConsumerDaemon, db_workqueue
-from rucio.db.sqla.constants import RequestState, TransferLimitDirection
+from rucio.db.sqla.constants import RequestState
 
 if TYPE_CHECKING:
     from collections.abc import Iterator

rucio/daemons/follower/follower.py

@@ -38,7 +38,7 @@ def aggregate_events(
         once: bool = False
 ) -> None:
     """
-    Collect all the events affecting the dids followed by the corresponding account.
+    Collect all the events affecting the DIDs followed by the corresponding account.
     """
 
     logging.info('event_aggregation: started')

rucio/daemons/hermes/hermes.py

@@ -115,7 +115,7 @@ def setup_activemq(
     brokers_resolved = []
     try:
         brokers_alias = config_get_list("messaging-hermes", "brokers")
-    except:
+    except Exception:
         raise Exception("Could not load brokers from configuration")
 
     logger(logging.INFO, "[broker] Resolving broker dns alias: %s", brokers_alias)
@@ -148,7 +148,7 @@ def setup_activemq(
     use_ssl = True
     try:
         use_ssl = config_get_bool("messaging-hermes", "use_ssl")
-    except:
+    except Exception:
         logger(
             logging.INFO,
             "[broker] Could not find use_ssl in configuration -- please update your rucio.cfg",

rucio/daemons/judge/cleaner.py

@@ -34,7 +34,7 @@ from rucio.common.logging import setup_logging
 from rucio.core.monitor import MetricManager
 from rucio.core.rule import delete_rule, get_expired_rules
 from rucio.daemons.common import HeartbeatHandler, run_daemon
-from rucio.db.sqla.constants import ORACLE_CONNECTION_LOST_CONTACT_REGEX, ORACLE_RESOURCE_BUSY_REGEX
+from rucio.db.sqla.constants import MYSQL_LOCK_NOWAIT_REGEX, ORACLE_CONNECTION_LOST_CONTACT_REGEX, ORACLE_RESOURCE_BUSY_REGEX, PSQL_PSYCOPG_LOCK_NOT_AVAILABLE_REGEX
 from rucio.db.sqla.util import get_db_time
 
 if TYPE_CHECKING:
@@ -103,7 +103,7 @@ def run_once(
             delete_rule(rule_id=rule_id, nowait=True)
             logger(logging.DEBUG, 'deletion of %s took %f' % (rule_id, time.time() - start))
         except (DatabaseException, DatabaseError, UnsupportedOperation) as e:
-            if match(ORACLE_RESOURCE_BUSY_REGEX, str(e.args[0])):
+            if match(ORACLE_RESOURCE_BUSY_REGEX, str(e.args[0])) or match(PSQL_PSYCOPG_LOCK_NOT_AVAILABLE_REGEX, str(e.args[0])) or match(MYSQL_LOCK_NOWAIT_REGEX, str(e.args[0])):
                 paused_rules[rule_id] = datetime.utcnow() + timedelta(seconds=randint(600, 2400))  # noqa: S311
                 METRICS.counter('exceptions.{exception}').labels(exception='LocksDetected').inc()
                 logger(logging.WARNING, 'Locks detected for %s' % rule_id)

rucio/daemons/judge/evaluator.py

@@ -36,7 +36,7 @@ from rucio.common.types import InternalScope
 from rucio.core.monitor import MetricManager
 from rucio.core.rule import delete_updated_did, get_updated_dids, re_evaluate_did
 from rucio.daemons.common import HeartbeatHandler, run_daemon
-from rucio.db.sqla.constants import ORACLE_CONNECTION_LOST_CONTACT_REGEX, ORACLE_RESOURCE_BUSY_REGEX, ORACLE_UNIQUE_CONSTRAINT_VIOLATED_REGEX
+from rucio.db.sqla.constants import MYSQL_LOCK_NOWAIT_REGEX, ORACLE_CONNECTION_LOST_CONTACT_REGEX, ORACLE_RESOURCE_BUSY_REGEX, ORACLE_UNIQUE_CONSTRAINT_VIOLATED_REGEX, PSQL_PSYCOPG_LOCK_NOT_AVAILABLE_REGEX
 
 if TYPE_CHECKING:
     from types import FrameType
@@ -52,7 +52,7 @@ def re_evaluator(
         did_limit: int = 100
 ) -> None:
     """
-    Main loop to check the re-evaluation of dids.
+    Main loop to check the re-evaluation of DIDs.
     """
 
     paused_dids = {}  # {(scope, name): datetime}
@@ -81,13 +81,13 @@ def run_once(
     # heartbeat
     start = time.time()  # NOQA
 
-    # Refresh paused dids
+    # Refresh paused DIDs
     iter_paused_dids = copy.copy(paused_dids)
     for key in iter_paused_dids:
         if datetime.utcnow() > paused_dids[key]:
             del paused_dids[key]
 
-    # Select a bunch of dids for re evaluation for this worker
+    # Select a bunch of DIDs for re-evaluation for this worker
     dids = get_updated_dids(total_workers=total_workers,
                             worker_number=worker_number,
                             limit=did_limit,
@@ -106,7 +106,7 @@ def run_once(
         if graceful_stop.is_set():
             break
 
-        # Check if this did has already been operated on
+        # Check if this DID has already been operated on
         did_tag = '%s:%s' % (did.scope.internal, did.name)
         if did_tag in done_dids:
             if did.rule_evaluation_action in done_dids[did_tag]:
@@ -116,7 +116,7 @@ def run_once(
         else:
             done_dids[did_tag] = []
 
-        # Jump paused dids
+        # Jump paused DIDs
         if (did.scope.internal, did.name) in paused_dids:
             continue
 
@@ -129,7 +129,7 @@ def run_once(
         except DataIdentifierNotFound:
             delete_updated_did(id_=did.id)
         except (DatabaseException, DatabaseError) as e:
-            if match(ORACLE_UNIQUE_CONSTRAINT_VIOLATED_REGEX, str(e.args[0])) or match(ORACLE_RESOURCE_BUSY_REGEX, str(e.args[0])):
+            if match(ORACLE_UNIQUE_CONSTRAINT_VIOLATED_REGEX, str(e.args[0])) or match(ORACLE_RESOURCE_BUSY_REGEX, str(e.args[0])) or match(PSQL_PSYCOPG_LOCK_NOT_AVAILABLE_REGEX, str(e.args[0])) or match(MYSQL_LOCK_NOWAIT_REGEX, str(e.args[0])):
                 paused_dids[(did.scope.internal, did.name)] = datetime.utcnow() + timedelta(seconds=randint(60, 600))  # noqa: S311
                 logger(logging.WARNING, 'Locks detected for %s:%s', did.scope, did.name)
                 METRICS.counter('exceptions.{exception}').labels(exception='LocksDetected').inc()