rucio 32.8.6__py3-none-any.whl → 35.8.0__py3-none-any.whl

rucio/core/permission/generic_multi_vo.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -16,7 +15,8 @@
 from typing import TYPE_CHECKING
 
 import rucio.core.scope
-from rucio.core.account import list_account_attributes, has_account_attribute
+from rucio.common.constants import RseAttr
+from rucio.core.account import has_account_attribute, list_account_attributes
 from rucio.core.identity import exist_identity_account
 from rucio.core.lifetime_exception import list_exceptions
 from rucio.core.rse import list_rse_attributes
@@ -26,7 +26,9 @@ from rucio.db.sqla.constants import IdentityType
 
 if TYPE_CHECKING:
     from typing import Optional
+
     from sqlalchemy.orm import Session
+
     from rucio.common.types import InternalAccount
 
 
@@ -832,7 +834,7 @@ def perm_set_local_account_limit(issuer, kwargs, *, session: "Optional[Session]"
     for kv in list_account_attributes(account=issuer, session=session):
         if kv['key'].startswith('country-') and kv['value'] == 'admin':
             admin_in_country.append(kv['key'].partition('-')[2])
-    if admin_in_country and list_rse_attributes(rse_id=kwargs['rse_id'], session=session).get('country') in admin_in_country:
+    if admin_in_country and list_rse_attributes(rse_id=kwargs['rse_id'], session=session).get(RseAttr.COUNTRY) in admin_in_country:
         return True
     return False
 
@@ -853,7 +855,7 @@ def perm_set_global_account_limit(issuer, kwargs, *, session: "Optional[Session]
     for kv in list_account_attributes(account=issuer, session=session):
         if kv['key'].startswith('country-') and kv['value'] == 'admin':
             admin_in_country.add(kv['key'].partition('-')[2])
-    resolved_rse_countries = {list_rse_attributes(rse_id=rse['rse_id'], session=session).get('country')
+    resolved_rse_countries = {list_rse_attributes(rse_id=rse['rse_id'], session=session).get(RseAttr.COUNTRY)
                               for rse in parse_expression(kwargs['rse_expression'], filter_={'vo': issuer.vo}, session=session)}
     if resolved_rse_countries.issubset(admin_in_country):
         return True
@@ -876,7 +878,7 @@ def perm_delete_local_account_limit(issuer, kwargs, *, session: "Optional[Sessio
     for kv in list_account_attributes(account=issuer, session=session):
         if kv['key'].startswith('country-') and kv['value'] == 'admin':
             admin_in_country.append(kv['key'].partition('-')[2])
-    if admin_in_country and list_rse_attributes(rse_id=kwargs['rse_id'], session=session).get('country') in admin_in_country:
+    if admin_in_country and list_rse_attributes(rse_id=kwargs['rse_id'], session=session).get(RseAttr.COUNTRY) in admin_in_country:
         return True
     return False
 
@@ -898,7 +900,7 @@ def perm_delete_global_account_limit(issuer, kwargs, *, session: "Optional[Sessi
         if kv['key'].startswith('country-') and kv['value'] == 'admin':
             admin_in_country.add(kv['key'].partition('-')[2])
     if admin_in_country:
-        resolved_rse_countries = {list_rse_attributes(rse_id=rse['rse_id'], session=session).get('country')
+        resolved_rse_countries = {list_rse_attributes(rse_id=rse['rse_id'], session=session).get(RseAttr.COUNTRY)
                                   for rse in parse_expression(kwargs['rse_expression'], filter_={'vo': issuer.vo}, session=session)}
         if resolved_rse_countries.issubset(admin_in_country):
             return True
@@ -952,7 +954,7 @@ def perm_get_global_account_usage(issuer, kwargs, *, session: "Optional[Session]
     for kv in list_account_attributes(account=issuer, session=session):
         if kv['key'].startswith('country-') and kv['value'] == 'admin':
             admin_in_country.add(kv['key'].partition('-')[2])
-    resolved_rse_countries = {list_rse_attributes(rse_id=rse['rse_id'], session=session).get('country')
+    resolved_rse_countries = {list_rse_attributes(rse_id=rse['rse_id'], session=session).get(RseAttr.COUNTRY)
                               for rse in parse_expression(kwargs['rse_exp'], filter_={'vo': issuer.vo}, session=session)}
 
     if resolved_rse_countries.issubset(admin_in_country):

rucio/core/quarantined_replica.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,13 +13,14 @@
 # limitations under the License.
 
 import datetime
-from typing import TYPE_CHECKING
+from collections.abc import Iterable
+from typing import TYPE_CHECKING, Any, Optional
 
-from sqlalchemy import and_, or_
+from sqlalchemy import and_, delete, or_, select
 from sqlalchemy.sql.expression import false, insert
 
 from rucio.common.utils import chunks
-from rucio.db.sqla import models, filter_thread_work
+from rucio.db.sqla import filter_thread_work, models
 from rucio.db.sqla.session import read_session, transactional_session
 
 if TYPE_CHECKING:
@@ -28,7 +28,7 @@ if TYPE_CHECKING:
 
 
 @transactional_session
-def add_quarantined_replicas(rse_id, replicas, *, session: "Session"):
+def add_quarantined_replicas(rse_id: str, replicas: list[dict[str, Any]], *, session: "Session") -> None:
     """
     Bulk add quarantined file replicas.
 
@@ -37,7 +37,7 @@ def add_quarantined_replicas(rse_id, replicas, *, session: "Session"):
     :param session:  The database session in use.
     """
 
-    # Exlude files that have a registered replica.  This is a
+    # Exclude files that have a registered replica.  This is a
     # safeguard against potential issues in the Auditor.
     file_clause = []
     for replica in replicas:
@@ -46,12 +46,18 @@ def add_quarantined_replicas(rse_id, replicas, *, session: "Session"):
                                     models.RSEFileAssociation.name == replica['name'],
                                     models.RSEFileAssociation.rse_id == rse_id))
     if file_clause:
-        file_query = session.query(models.RSEFileAssociation.scope,
-                                   models.RSEFileAssociation.name,
-                                   models.RSEFileAssociation.rse_id).\
-            with_hint(models.RSEFileAssociation, "index(REPLICAS REPLICAS_PK)", 'oracle').\
-            filter(or_(*file_clause))
-        existing_replicas = [(scope, name, rseid) for scope, name, rseid in file_query]
+        stmt = select(
+            models.RSEFileAssociation.scope,
+            models.RSEFileAssociation.name,
+            models.RSEFileAssociation.rse_id
+        ).with_hint(
+            models.RSEFileAssociation,
+            'INDEX(REPLICAS REPLICAS_PK)',
+            'oracle'
+        ).where(
+            or_(*file_clause)
+        )
+        existing_replicas = [(scope, name, rseid) for scope, name, rseid in session.execute(stmt).all()]
         replicas = [replica for replica in replicas if (replica.get('scope', None), replica.get('name', None), rse_id) not in existing_replicas]
 
     # Exclude files that have already been added to the quarantined
@@ -60,29 +66,34 @@ def add_quarantined_replicas(rse_id, replicas, *, session: "Session"):
     for replica in replicas:
         quarantine_clause.append(and_(models.QuarantinedReplica.path == replica['path'],
                                       models.QuarantinedReplica.rse_id == rse_id))
-    quarantine_query = session.query(models.QuarantinedReplica.path,
-                                     models.QuarantinedReplica.rse_id).\
-        filter(or_(*quarantine_clause))
-    quarantine_replicas = [(path, rseid) for path, rseid in quarantine_query]
+    stmt = select(
+        models.QuarantinedReplica.path,
+        models.QuarantinedReplica.rse_id
+    ).where(
+        or_(*quarantine_clause)
+    )
+    quarantine_replicas = [(path, rseid) for path, rseid in session.execute(stmt).all()]
     replicas = [replica for replica in replicas if (replica['path'], rse_id) not in quarantine_replicas]
 
-    session.execute(
-        insert(models.QuarantinedReplica),
-        [{'rse_id': rse_id,
-          'path': file['path'],
-          'scope': file.get('scope'),
-          'name': file.get('name'),
-          'bytes': file.get('bytes')}
-         for file in replicas])
+    values = [{'rse_id': rse_id,
+               'path': replica['path'],
+               'scope': replica.get('scope'),
+               'name': replica.get('name'),
+               'bytes': replica.get('bytes')}
+              for replica in replicas]
+    stmt = insert(
+        models.QuarantinedReplica
+    )
+    session.execute(stmt, values)
 
 
 @transactional_session
-def delete_quarantined_replicas(rse_id, replicas, *, session: "Session"):
+def delete_quarantined_replicas(rse_id: str, replicas: Iterable[dict[str, Any]], *, session: "Session") -> None:
     """
     Delete file replicas.
 
     :param rse_id: the rse id.
-    :param replicas: A list of dicts with the replica information.
+    :param replicas: An iterable of dicts with the replica information.
     :param session: The database session in use.
     """
 
@@ -91,24 +102,30 @@ def delete_quarantined_replicas(rse_id, replicas, *, session: "Session"):
         conditions.append(models.QuarantinedReplica.path == replica['path'])
 
     if conditions:
-        session.query(models.QuarantinedReplica).\
-            filter(models.QuarantinedReplica.rse_id == rse_id).\
-            filter(or_(*conditions)).\
-            delete(synchronize_session=False)
+        stmt = delete(
+            models.QuarantinedReplica
+        ).where(
+            and_(models.QuarantinedReplica.rse_id == rse_id,
+                 or_(*conditions))
+        ).execution_options(
+            synchronize_session=False
+        )
+        session.execute(stmt)
 
     if replicas:
-        session.execute(
-            insert(models.QuarantinedReplicaHistory),
-            [{'rse_id': rse_id, 'path': replica['path'],
-              'bytes': replica.get('bytes'),
-              'created_at': replica.get('created_at'),
-              'deleted_at': datetime.datetime.utcnow()}
-             for replica in replicas]
+        values = [{'rse_id': rse_id, 'path': replica['path'],
+                   'bytes': replica.get('bytes'),
+                   'created_at': replica.get('created_at'),
+                   'deleted_at': datetime.datetime.utcnow()}
+                  for replica in replicas]
+        stmt = insert(
+            models.QuarantinedReplicaHistory
         )
+        session.execute(stmt, values)
 
 
 @read_session
-def list_quarantined_replicas(rse_id, limit, worker_number=None, total_workers=None, *, session: "Session"):
+def list_quarantined_replicas(rse_id: str, limit: int, worker_number: Optional[int] = None, total_workers: Optional[int] = None, *, session: "Session") -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
     """
     List RSE Quarantined File replicas.
 
@@ -127,28 +144,38 @@ def list_quarantined_replicas(rse_id, limit, worker_number=None, total_workers=N
     quarantined_replicas = {}
     real_replicas = []
     dark_replicas = []
-    query = session.query(models.QuarantinedReplica.path,
-                          models.QuarantinedReplica.bytes,
-                          models.QuarantinedReplica.scope,
-                          models.QuarantinedReplica.name,
-                          models.QuarantinedReplica.created_at).\
-        filter(models.QuarantinedReplica.rse_id == rse_id)
-    query = filter_thread_work(session=session, query=query, total_threads=total_workers, thread_id=worker_number, hash_variable='path')
-
-    for path, bytes_, scope, name, created_at in query.limit(limit):
+    stmt = select(
+        models.QuarantinedReplica.path,
+        models.QuarantinedReplica.bytes,
+        models.QuarantinedReplica.scope,
+        models.QuarantinedReplica.name,
+        models.QuarantinedReplica.created_at
+    ).where(
+        models.QuarantinedReplica.rse_id == rse_id
+    )
+    stmt = filter_thread_work(session=session, query=stmt, total_threads=total_workers, thread_id=worker_number, hash_variable='path')
+    stmt = stmt.limit(
+        limit
+    )
+    for path, bytes_, scope, name, created_at in session.execute(stmt).all():
         if not (scope, name) in quarantined_replicas:
             quarantined_replicas[(scope, name)] = []
             replicas_clause.append(and_(models.RSEFileAssociation.scope == scope,
                                         models.RSEFileAssociation.name == name))
         quarantined_replicas[(scope, name)].append((path, bytes_, created_at))
 
+    stmt = select(
+        models.RSEFileAssociation.scope,
+        models.RSEFileAssociation.name
+    ).where(
+        models.RSEFileAssociation.rse_id == rse_id
+    )
     for chunk in chunks(replicas_clause, 20):
-        query = session.query(models.RSEFileAssociation.scope,
-                              models.RSEFileAssociation.name).\
-            filter(models.RSEFileAssociation.rse_id == rse_id).\
-            filter(or_(*chunk))
+        curr_stmt = stmt.where(
+            or_(*chunk)
+        )
 
-        for scope, name in query.all():
+        for scope, name in session.execute(curr_stmt).all():
             reps = quarantined_replicas.pop((scope, name))
             real_replicas.extend([{'scope': scope,
                                    'name': name,
@@ -171,7 +198,7 @@ def list_quarantined_replicas(rse_id, limit, worker_number=None, total_workers=N
 
 
 @read_session
-def list_rses_with_quarantined_replicas(filters=None, *, session: "Session"):
+def list_rses_with_quarantined_replicas(filters: Optional[dict[str, Any]] = None, *, session: "Session") -> list[str]:
     """
     List RSEs in the Quarantined Queues.
 
@@ -180,11 +207,17 @@ def list_rses_with_quarantined_replicas(filters=None, *, session: "Session"):
 
     :returns: a list of RSEs.
     """
-    query = session.query(models.RSE.id).distinct(models.RSE.id).\
-        filter(models.QuarantinedReplica.rse_id == models.RSE.id).\
-        filter(models.RSE.deleted == false())
+    stmt = select(
+        models.RSE.id
+    ).distinct(
+    ).where(
+        and_(models.QuarantinedReplica.rse_id == models.RSE.id,
+             models.RSE.deleted == false())
+    )
 
     if filters and filters.get('vo'):
-        query = query.filter(getattr(models.RSE, 'vo') == filters.get('vo'))
+        stmt = stmt.where(
+            models.RSE.vo == filters.get('vo')
+        )
 
-    return [rse for (rse,) in query]
+    return [str(rseid) for rseid in session.execute(stmt).scalars().all()]