rucio 37.5.0__py3-none-any.whl → 37.7.0__py3-none-any.whl

rucio/daemons/transmogrifier/transmogrifier.py

@@ -124,7 +124,7 @@ def __split_rule_select_rses(
     weight: int,
     rse_expression: str,
     copies: int,
-    blocklisted_rse_id: list,
+    blocklisted_rse_ids: list,
     logger: LoggerFunction,
 ) -> tuple[list, bool, bool]:
     """
@@ -138,7 +138,7 @@ def __split_rule_select_rses(
     :param weight: The weight of the rule.
     :param rse_expression: The RSE expression of the rule.
     :param copies: The number of copies.
-    :param blocklisted_rse_id: The list of blocklisted_rse_id.
+    :param blocklisted_rse_ids: The list of blocklisted rse_ids.
     :param logger: The logger.
     :return: A tuple with list selected_rses, and 2 booleans create_rule, wont_reevaluate.
     """
@@ -172,7 +172,7 @@ def __split_rule_select_rses(
                 copies=copies,
                 size=0,
                 preferred_rses=preferred_rses,
-                blocklist=blocklisted_rse_id,
+                blocklist=blocklisted_rse_ids,
             )
             wont_reevaluate = True
             break
@@ -184,26 +184,22 @@ def __split_rule_select_rses(
         ) as error:
             logger(
                 logging.WARNING,
-                'Problem getting RSEs for subscription "%s" for account %s : %s. %s'
-                % (
-                    subscription_name,
-                    account,
-                    str(error),
-                    'Try including blocklisted sites' if attempt == 0 else 'Skipping rule creation.'
-                ),
+                'Problem getting RSEs for subscription "%s" for account %s : %s. %s',
+                subscription_name,
+                account,
+                str(error),
+                'Try including blocklisted sites' if attempt == 0 else 'Skipping rule creation.'
             )
             # Now including the blocklisted sites
-            blocklisted_rse_id = []
+            blocklisted_rse_ids = []
             METRICS.counter(name="addnewrule.errortype.{exception}").labels(exception=str(error.__class__.__name__)).inc()
             wont_reevaluate = True
         except Exception as error:
             logger(
                 logging.ERROR,
-                "Problem resolving RSE expression %s : %s"
-                % (
-                    rse_expression,
-                    str(error),
-                )
+                "Problem resolving RSE expression %s : %s",
+                rse_expression,
+                str(error),
             )
     if len(preferred_rses) - len(preferred_unmatched) >= copies:
         create_rule = False
@@ -242,6 +238,7 @@ def get_subscriptions(logger: LoggerFunction = logging.log) -> list[dict]:
                     break
                 if rule.get("copies") == "*":
                     rule["copies"] = len(list_rses_from_expression)
+                    rule["wildcard"] = True
                     overwrite_rules = True
             if skip_sub:
                 continue
@@ -272,7 +269,7 @@ def get_subscriptions(logger: LoggerFunction = logging.log) -> list[dict]:
             subscriptions.extend(sub_dict[priority])
         logger(logging.INFO, "%i active subscriptions", len(subscriptions))
     except SubscriptionNotFound as error:
-        logger(logging.WARNING, "No subscriptions defined: %s" % (str(error)))
+        logger(logging.WARNING, "No subscriptions defined: %s", (str(error)))
         return []
     except TypeError as error:
         logger(
@@ -307,7 +304,7 @@ def __is_matching_subscription(
     try:
         filter_string = loads(subscription["filter"])
     except ValueError as error:
-        logging.error("%s : Subscription will be skipped" % error)
+        logging.error("%s : Subscription will be skipped", error)
         return False
     # Loop over the keys of filter_string for subscription
     for key in filter_string:
@@ -399,7 +396,7 @@ def select_algorithm(
     selected_rses = {}
     for rule_id in rule_ids:
         rule = get_rule(rule_id)
-        logging.debug("In select_algorithm, %s", str(rule))
+        logger(logging.DEBUG, "In select_algorithm, %s", str(rule))
         rse = rule["rse_expression"]
         vo = rule["account"].vo
         if rse_exists(rse, vo=vo):
@@ -439,7 +436,7 @@ def select_algorithm(
                     weight=rule.get("weight"),
                     rse_expression=rse_expression,
                     copies=rule.get('copies'),
-                    blocklisted_rse_id=params['blocklisted_rse_id'],
+                    blocklisted_rse_ids=params['blocklisted_rse_ids'],
                     logger=logger,
                 )
                 dict_selected_rses = {}
@@ -487,12 +484,13 @@ def run_once(heartbeat_handler: "HeartbeatHandler", bulk: int, **_kwargs) -> boo
 
     worker_number, total_workers, logger = heartbeat_handler.live()
     stopwatch = Stopwatch()
-    blocklisted_rse_id = [rse["id"] for rse in list_rses({"availability_write": False})]
+    block_listed = {rse['rse']: rse["id"] for rse in list_rses({"availability_write": False})}
+    blocklisted_rse_ids = list(block_listed.values())
     identifiers = []
     #  List all the active subscriptions
     subscriptions = get_subscriptions(logger=logger)
 
-    #  Loop over all the new dids
+    #  Loop over all the new DIDs
     #  Get the new DIDs based on the is_new flag
     logger(logging.DEBUG, "Listing new dids")
     for did in list_new_dids(
@@ -535,6 +533,7 @@ def run_once(heartbeat_handler: "HeartbeatHandler", bulk: int, **_kwargs) -> boo
                     #  Get all the rule and subscription parameters
                     rule_dict = __get_rule_dict(rule_dict, subscription)
                     weight = rule_dict.get("weight", None)
+                    ignore_availability = rule_dict.get("ignore_availability", False)
                     source_replica_expression = rule_dict.get(
                         "source_replica_expression", None
                     )
@@ -551,7 +550,7 @@ def run_once(heartbeat_handler: "HeartbeatHandler", bulk: int, **_kwargs) -> boo
                         params['rse_expression'] = rule_dict.get("rse_expression")
                         params['subscription_id'] = subscription["id"]
                         params['subscription_name'] = subscription["name"]
-                        params['blocklisted_rse_id'] = blocklisted_rse_id
+                        params['blocklisted_rse_ids'] = blocklisted_rse_ids
                         if rule_dict.get("associated_site_idx", None):
                             params["associated_site_idx"] = rule_dict.get(
                                 "associated_site_idx", None
@@ -583,7 +582,7 @@ def run_once(heartbeat_handler: "HeartbeatHandler", bulk: int, **_kwargs) -> boo
                             weight=weight,
                             rse_expression=rule_dict.get("rse_expression"),
                             copies=copies,
-                            blocklisted_rse_id=blocklisted_rse_id,
+                            blocklisted_rse_ids=blocklisted_rse_ids,
                             logger=logger,
                         )
                         copies = 1
@@ -595,20 +594,26 @@ def run_once(heartbeat_handler: "HeartbeatHandler", bulk: int, **_kwargs) -> boo
                     nb_rule = 0
                     #  Try to create the rule
                     logger(logging.DEBUG, 'selected_rses : %s' % selected_rses)
-                    try:
-                        for rse in selected_rses:
-                            if isinstance(selected_rses, dict):
-                                #  selected_rses is a dictionary only when split_rule is True or for chained subscriptions
-                                source_replica_expression = selected_rses[rse].get(
-                                    "source_replica_expression",
-                                    None,
-                                )
-                                weight = selected_rses[rse].get("weight", None)
-                            logger(
-                                logging.INFO,
-                                "Will insert one rule for %s:%s on %s"
-                                % (did["scope"], did["name"], rse),
+                    for rse in selected_rses:
+                        if isinstance(selected_rses, dict):
+                            #  selected_rses is a dictionary only when split_rule is True or for chained subscriptions
+                            source_replica_expression = selected_rses[rse].get(
+                                "source_replica_expression",
+                                None,
                             )
+                            weight = selected_rses[rse].get("weight", None)
+                        logger(
+                            logging.INFO,
+                            "Will insert one rule for %s:%s on %s",
+                            did["scope"], did["name"], rse,
+                        )
+                        if rse in block_listed and rule_dict.get("wildcard"):
+                            if ignore_availability:
+                                logger(logging.WARNING, "RSE %s is unavailable, but wildcard number of copies is used with ignore_availability option. Creating a rule", rse)
+                            else:
+                                logger(logging.INFO, "RSE %s is unavailable and wildcard number of copies is used. Skipping rule creation", rse)
+                                continue
+                        try:
                             rule_ids = add_rule(
                                 dids=[
                                     {
@@ -627,9 +632,7 @@ def run_once(heartbeat_handler: "HeartbeatHandler", bulk: int, **_kwargs) -> boo
                                 source_replica_expression=source_replica_expression,
                                 activity=rule_dict.get("activity"),
                                 purge_replicas=rule_dict.get("purge_replicas", False),
-                                ignore_availability=rule_dict.get(
-                                    "ignore_availability", None
-                                ),
+                                ignore_availability=ignore_availability,
                                 comment=rule_dict.get("comment"),
                                 delay_injection=rule_dict.get("delay_injection"),
                             )
@@ -640,41 +643,40 @@ def run_once(heartbeat_handler: "HeartbeatHandler", bulk: int, **_kwargs) -> boo
                             if split_rule:
                                 success = True
 
-                        METRICS.counter("addnewrule.done").inc(nb_rule)
-                        METRICS.counter("addnewrule.activity.{activity}").labels(activity="".join(rule_dict.get("activity").split())).inc(nb_rule)
-                        success = True
-                    except (
-                        InvalidReplicationRule,
-                        InvalidRuleWeight,
-                        InvalidRSEExpression,
-                        StagingAreaRuleRequiresLifetime,
-                        DuplicateRule,
-                    ) as error:
-                        # Errors that won't be retried
-                        success = True
-                        logger(logging.ERROR, str(error))
-                        METRICS.counter("addnewrule.errortype.{exception}").labels(exception=str(error.__class__.__name__)).inc()
-                    except Exception:
-                        # Errors that will be retried
-                        METRICS.counter("addnewrule.errortype.{exception}").labels(exception="unknown").inc()
-                        logger(logging.ERROR, "Unexpected error", exc_info=True)
+                        except (
+                            InvalidReplicationRule,
+                            InvalidRuleWeight,
+                            InvalidRSEExpression,
+                            StagingAreaRuleRequiresLifetime,
+                            DuplicateRule,
+                        ) as error:
+                            # Errors that won't be retried
+                            success = True
+                            logger(logging.ERROR, str(error))
+                            METRICS.counter("addnewrule.errortype.{exception}").labels(exception=str(error.__class__.__name__)).inc()
+                        except Exception:
+                            # Errors that will be retried
+                            METRICS.counter("addnewrule.errortype.{exception}").labels(exception="unknown").inc()
+                            logger(logging.ERROR, "Unexpected error", exc_info=True)
+
+                    METRICS.counter("addnewrule.done").inc(nb_rule)
+                    METRICS.counter("addnewrule.activity.{activity}").labels(activity="".join(rule_dict.get("activity").split())).inc(nb_rule)
+                    success = True
 
                     did_success = did_success and success
                     if not success:
                         logger(
                             logging.ERROR,
-                            "Rule for %s:%s on %s cannot be inserted"
-                            % (
-                                did["scope"],
-                                did["name"],
-                                rule_dict.get("rse_expression"),
-                            ),
+                            "Rule for %s:%s on %s cannot be inserted",
+                            did["scope"],
+                            did["name"],
+                            rule_dict.get("rse_expression"),
                         )
                     else:
                         logger(
                             logging.INFO,
-                            "%s rule(s) inserted in %f seconds"
-                            % (str(nb_rule), time.time() - stime),
+                            "%s rule(s) inserted in %f seconds",
+                            str(nb_rule), time.time() - stime,
                         )
 
         if did_success:
@@ -697,7 +699,7 @@ def run_once(heartbeat_handler: "HeartbeatHandler", bulk: int, **_kwargs) -> boo
     flag_stopwatch = Stopwatch()
     for identifier in chunks(identifiers, 100):
         set_new_dids(identifier, None)
-    logger(logging.DEBUG, "Time to set the new flag : %f" % flag_stopwatch.elapsed)
+    logger(logging.DEBUG, "Time to set the new flag : %f", flag_stopwatch.elapsed)
 
     stopwatch.stop()
 
@@ -709,9 +711,9 @@ def run_once(heartbeat_handler: "HeartbeatHandler", bulk: int, **_kwargs) -> boo
         )
     logger(
         logging.INFO,
-        "It took %f seconds to process %i DIDs" % (stopwatch.elapsed, len(identifiers)),
+        "It took %f seconds to process %i DIDs", stopwatch.elapsed, len(identifiers),
     )
-    logger(logging.DEBUG, "DIDs processed : %s" % (str(identifiers)))
+    logger(logging.DEBUG, "DIDs processed : %s", str(identifiers))
     METRICS.counter(name="transmogrifier.job.done").inc(1)
     METRICS.timer("job.duration").observe(stopwatch.elapsed)
     must_sleep = True

rucio/daemons/undertaker/undertaker.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 '''
-Undertaker is a daemon to manage expired did.
+Undertaker is a daemon to manage expired DID.
 '''
 
 import functools
@@ -29,6 +29,7 @@ from typing import TYPE_CHECKING
 from sqlalchemy.exc import DatabaseError
 
 import rucio.db.sqla.util
+from rucio.common.constants import DEFAULT_VO
 from rucio.common.exception import DatabaseException, RuleNotFound, UnsupportedOperation
 from rucio.common.logging import setup_logging
 from rucio.common.types import InternalAccount
@@ -36,7 +37,7 @@ from rucio.common.utils import chunks
 from rucio.core.did import delete_dids, list_expired_dids
 from rucio.core.monitor import MetricManager
 from rucio.daemons.common import HeartbeatHandler, run_daemon
-from rucio.db.sqla.constants import MYSQL_LOCK_NOWAIT_REGEX, ORACLE_RESOURCE_BUSY_REGEX, PSQL_LOCK_NOT_AVAILABLE_REGEX
+from rucio.db.sqla.constants import MYSQL_LOCK_NOWAIT_REGEX, ORACLE_RESOURCE_BUSY_REGEX, PSQL_LOCK_NOT_AVAILABLE_REGEX, PSQL_PSYCOPG_LOCK_NOT_AVAILABLE_REGEX
 
 if TYPE_CHECKING:
     from types import FrameType
@@ -51,7 +52,7 @@ DAEMON_NAME = 'undertaker'
 
 def undertaker(once: bool = False, sleep_time: int = 60, chunk_size: int = 10) -> None:
     """
-    Main loop to select and delete dids.
+    Main loop to select and delete DIDs.
     """
     paused_dids = {}  # {(scope, name): datetime}
     run_daemon(
@@ -72,7 +73,7 @@ def run_once(paused_dids: dict[tuple, datetime], chunk_size: int, heartbeat_hand
     worker_number, total_workers, logger = heartbeat_handler.live()
 
     try:
-        # Refresh paused dids
+        # Refresh paused DIDs
         iter_paused_dids = deepcopy(paused_dids)
         for key in iter_paused_dids:
             if datetime.utcnow() > paused_dids[key]:
@@ -90,20 +91,20 @@ def run_once(paused_dids: dict[tuple, datetime], chunk_size: int, heartbeat_hand
             _, _, logger = heartbeat_handler.live()
             try:
                 logger(logging.INFO, 'Receive %s dids to delete', len(chunk))
-                delete_dids(dids=chunk, account=InternalAccount('root', vo='def'), expire_rules=True)
+                delete_dids(dids=chunk, account=InternalAccount('root', vo=DEFAULT_VO), expire_rules=True)
                 logger(logging.INFO, 'Delete %s dids', len(chunk))
                 METRICS.counter(name='undertaker.delete_dids').inc(len(chunk))
             except RuleNotFound as error:
                 logger(logging.ERROR, error)
             except (DatabaseException, DatabaseError, UnsupportedOperation) as e:
-                if match(ORACLE_RESOURCE_BUSY_REGEX, str(e.args[0])) or match(PSQL_LOCK_NOT_AVAILABLE_REGEX, str(e.args[0])) or match(MYSQL_LOCK_NOWAIT_REGEX, str(e.args[0])):
+                if match(ORACLE_RESOURCE_BUSY_REGEX, str(e.args[0])) or match(PSQL_LOCK_NOT_AVAILABLE_REGEX, str(e.args[0])) or match(PSQL_PSYCOPG_LOCK_NOT_AVAILABLE_REGEX, str(e.args[0])) or match(MYSQL_LOCK_NOWAIT_REGEX, str(e.args[0])):
                     for did in chunk:
                         paused_dids[(did['scope'], did['name'])] = datetime.utcnow() + timedelta(seconds=randint(600, 2400))  # noqa: S311
                     METRICS.counter('delete_dids.exceptions.{exception}').labels(exception='LocksDetected').inc()
                     logger(logging.WARNING, 'Locks detected for chunk')
                 else:
                     logger(logging.ERROR, 'Got database error %s.', str(e))
-    except:
+    except Exception:
         logging.critical(traceback.format_exc())
 
 

rucio/db/sqla/constants.py

@@ -26,6 +26,7 @@ ORACLE_DEADLOCK_DETECTED_REGEX = r".*ORA-00060.*"
 ORACLE_RESOURCE_BUSY_REGEX = r".*ORA-00054.*"
 ORACLE_UNIQUE_CONSTRAINT_VIOLATED_REGEX = r".*ORA-00001.*"
 PSQL_LOCK_NOT_AVAILABLE_REGEX = r".*55P03.*"
+PSQL_PSYCOPG_LOCK_NOT_AVAILABLE_REGEX = r".*psycopg.errors.LockNotAvailable.*"
 MYSQL_LOCK_NOWAIT_REGEX = r".*3572.*"
 MYSQL_LOCK_WAIT_TIMEOUT_EXCEEDED = "ERROR 1205 (HY000)"
 
@@ -196,9 +197,9 @@ class SubscriptionState(Enum):
     BROKEN = 'B'
 
 
-#class TransferLimitDirection(Enum):
-#    SOURCE = 'S'
-#    DESTINATION = 'D'
+# class TransferLimitDirection(Enum):
+#     SOURCE = 'S'
+#     DESTINATION = 'D'
 
 
 class DatabaseOperationType(Enum):

rucio/db/sqla/migrate_repo/versions/3082b8cef557_add_naming_convention_table_and_closed_.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-''' add convention table and closed_at to dids '''
+''' add convention table and closed_at to DIDs '''
 
 import datetime
 

rucio/db/sqla/migrate_repo/versions/4df2c5ddabc0_remove_temporary_dids.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-''' remove temporary dids '''
+''' remove temporary DIDs '''
 
 import sqlalchemy as sa
 from alembic import context

rucio/db/sqla/migrate_repo/versions/c129ccdb2d5_add_lumiblocknr_to_dids.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-''' add lumiblocknr to dids '''
+''' add lumiblocknr to DIDs '''
 
 import sqlalchemy as sa
 from alembic import context

rucio/db/sqla/models.py

@@ -29,7 +29,7 @@ from sqlalchemy.types import LargeBinary
 # and it must be renamed to avoid conflicts with the policy package schema modules
 from rucio.common import schema as common_schema
 from rucio.common import utils
-from rucio.common.constants import TransferLimitDirection
+from rucio.common.constants import DEFAULT_VO, TransferLimitDirection
 from rucio.common.types import InternalAccount, InternalScope  # noqa: TCH001 (types are needed by SQLAlchemy)
 from rucio.db.sqla.constants import (
     AccountStatus,
@@ -563,7 +563,7 @@ class DeletedDataIdentifier(BASE, ModelBase):
 
 
 class UpdatedDID(BASE, ModelBase):
-    """Represents the recently updated dids"""
+    """Represents the recently updated DIDs"""
     __tablename__ = 'updated_dids'
     id: Mapped[str] = mapped_column(GUID(), default=utils.generate_uuid)
     scope: Mapped[InternalScope] = mapped_column(InternalScopeString(common_schema.get_schema_value('SCOPE_LENGTH')))
@@ -776,7 +776,7 @@ class RSE(BASE, SoftModelBase):
     __tablename__ = 'rses'
     id: Mapped[str] = mapped_column(GUID(), default=utils.generate_uuid)
     rse: Mapped[str] = mapped_column(String(255))
-    vo: Mapped[str] = mapped_column(String(3), nullable=False, server_default='def')
+    vo: Mapped[str] = mapped_column(String(3), nullable=False, server_default=DEFAULT_VO)
     rse_type: Mapped[RSEType] = mapped_column(Enum(RSEType, name='RSES_TYPE_CHK',
                                                    create_constraint=True,
                                                    values_callable=lambda obj: [e.value for e in obj]),

rucio/db/sqla/session.py

@@ -54,7 +54,7 @@ if TYPE_CHECKING:
 try:
     main_script = os.path.basename(sys.argv[0])
     CURRENT_COMPONENT = main_script.split('-')[1]
-except:
+except Exception:
     CURRENT_COMPONENT = None
 
 DATABASE_SECTION = 'database'
@@ -63,7 +63,7 @@ try:
         sql_connection = config_get('%s-database' % CURRENT_COMPONENT, 'default', check_config_table=False).strip()
         if sql_connection and len(sql_connection):
             DATABASE_SECTION = '%s-database' % CURRENT_COMPONENT
-except:
+except Exception:
     pass
 
 DEFAULT_SCHEMA_NAME = config_get(DATABASE_SECTION, 'schema',
@@ -221,7 +221,7 @@ def get_engine() -> 'Engine':
         for param, param_type in config_params:
             try:
                 params[param] = param_type(config_get(DATABASE_SECTION, param, check_config_table=False))
-            except:
+            except Exception:
                 pass
         _ENGINE = create_engine(sql_connection, **params)
         if 'mysql' in sql_connection:
@@ -406,7 +406,7 @@ def read_session(function: "Callable[P, R]"):
             except DatabaseError as error:
                 session.rollback()  # type: ignore
                 raise DatabaseException(str(error))
-            except:
+            except Exception:
                 session.rollback()  # type: ignore
                 raise
             finally:
@@ -451,7 +451,7 @@ def stream_session(function: "Callable[P, R]"):
             except DatabaseError as error:
                 session.rollback()  # type: ignore
                 raise DatabaseException(str(error))
-            except:
+            except Exception:
                 session.rollback()  # type: ignore
                 raise
             finally:
@@ -460,7 +460,7 @@ def stream_session(function: "Callable[P, R]"):
             try:
                 for row in function(*args, session=session, **kwargs):
                     yield row
-            except:
+            except Exception:
                 raise
     return _update_session_wrapper(new_funct, function)
 
@@ -491,7 +491,7 @@ def transactional_session(function: "Callable[P, R]") -> 'Callable':
             except DatabaseError as error:
                 session.rollback()  # type: ignore
                 raise DatabaseException(str(error))
-            except:
+            except Exception:
                 session.rollback()  # type: ignore
                 raise
             finally:

rucio/db/sqla/util.py

@@ -33,6 +33,7 @@ from sqlalchemy.sql.expression import select, text
 from rucio import alembicrevision
 from rucio.common.cache import MemcacheRegion
 from rucio.common.config import config_get, config_get_list
+from rucio.common.constants import DEFAULT_VO
 from rucio.common.schema import get_schema_value
 from rucio.common.types import InternalAccount, LoggerFunction
 from rucio.common.utils import generate_uuid
@@ -136,7 +137,7 @@ def create_base_vo() -> None:
 
     session_scoped = get_session()
 
-    vo = models.VO(vo='def', description='Default base VO', email='N/A')
+    vo = models.VO(vo=DEFAULT_VO, description='Default base VO', email='N/A')
     with session_scoped() as s:
         with s.begin():
             s.add_all([vo])
@@ -172,7 +173,7 @@ def create_root_account() -> None:
     else:
         access = 'root'
 
-    account = models.Account(account=InternalAccount(access, 'def'), account_type=AccountType.SERVICE, status=AccountStatus.ACTIVE)
+    account = models.Account(account=InternalAccount(access, DEFAULT_VO), account_type=AccountType.SERVICE, status=AccountStatus.ACTIVE)
 
     salt = urandom(255)
     salted_password = salt + up_pwd.encode()