rucio 32.8.6__py3-none-any.whl → 35.8.0__py3-none-any.whl

rucio/core/vo.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,13 +13,13 @@
 # limitations under the License.
 
 import re
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
 
-from sqlalchemy.exc import DatabaseError, IntegrityError
-from sqlalchemy.orm.exc import NoResultFound
+from sqlalchemy import select, update
+from sqlalchemy.exc import DatabaseError, IntegrityError, NoResultFound
 
 from rucio.common import exception
-from rucio.common.config import config_get_bool, config_get
+from rucio.common.config import config_get, config_get_bool
 from rucio.common.types import InternalAccount
 from rucio.db.sqla import models
 from rucio.db.sqla.constants import AccountType, IdentityType
@@ -34,7 +33,7 @@ if TYPE_CHECKING:
 
 
 @read_session
-def vo_exists(vo, *, session: "Session"):
+def vo_exists(vo: str, *, session: "Session") -> bool:
     """
     Verify that the vo exists.
 
@@ -43,17 +42,22 @@ def vo_exists(vo, *, session: "Session"):
 
     :returns: True if the vo is in the vo table, False otherwise
     """
-    return True if session.query(models.VO).filter_by(vo=vo).first() else False
+    stmt = select(
+        models.VO
+    ).where(
+        models.VO.vo == vo
+    )
+    return bool(session.execute(stmt).scalar())
 
 
 @transactional_session
-def add_vo(vo, description, email, *, session: "Session"):
+def add_vo(vo: str, description: str, email: str, *, session: "Session") -> None:
     """
     Add a VO and setup a new root user.
     New root user will have account name 'root' and a userpass identity with username: 'root@<vo>' and password: 'password'
 
     :param vo: 3-letter unique tag for a VO.
-    :param descrition: Descriptive string for the VO (e.g. Full name).
+    :param description: Descriptive string for the VO (e.g. Full name).
     :param email: Contact email for the VO.
     :param session: The db session in use.
     """
@@ -81,7 +85,7 @@ def add_vo(vo, description, email, *, session: "Session"):
                          account=new_root,
                          email=email,
                          default=False,
-                         password='password',
+                         password='password',  # noqa: S106
                          session=session)
 
     for ident in list_identities(account=InternalAccount('super_root', vo='def'), session=session):
@@ -89,7 +93,7 @@ def add_vo(vo, description, email, *, session: "Session"):
 
 
 @read_session
-def list_vos(*, session: "Session"):
+def list_vos(*, session: "Session") -> list[dict[str, Any]]:
     """
     List all the VOs in the db.
 
@@ -99,10 +103,12 @@ def list_vos(*, session: "Session"):
     if not config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
         raise exception.UnsupportedOperation('VO operations cannot be performed in single VO mode.')
 
-    query = session.query(models.VO)
+    stmt = select(
+        models.VO
+    )
 
     vos = []
-    for vo in query.all():
+    for vo in session.execute(stmt).scalars().all():
         vo_dict = {'vo': vo.vo,
                    'description': vo.description,
                    'email': vo.email,
@@ -114,7 +120,7 @@ def list_vos(*, session: "Session"):
 
 
 @transactional_session
-def update_vo(vo, parameters, *, session: "Session"):
+def update_vo(vo: str, parameters: dict[str, Any], *, session: "Session") -> None:
     """
     Update VO properties (email, description).
 
@@ -126,22 +132,34 @@ def update_vo(vo, parameters, *, session: "Session"):
         raise exception.UnsupportedOperation('VO operations cannot be performed in single VO mode.')
 
     try:
-        query = session.query(models.VO).filter_by(vo=vo).one()
+        stmt = select(
+            models.VO
+        ).where(
+            models.VO.vo == vo
+        )
+        session.execute(stmt).scalar_one()
     except NoResultFound:
         raise exception.VONotFound('VO {} not found'.format(vo))
     param = {}
     for key in parameters:
         if key in ['email', 'description']:
             param[key] = parameters[key]
-    query.update(param)
+    stmt = update(
+        models.VO
+    ).where(
+        models.VO.vo == vo
+    ).values(
+        param
+    )
+    session.execute(stmt)
 
 
-def map_vo(vo):
+def map_vo(vo: str) -> str:
     """
     Converts a long VO name into the internal short (three letter)
     tag mapping.
     Mappings are loaded from the vo-map section of the config database table.
-    If a mapping is not found, the orignal is returned unchanged.
+    If a mapping is not found, the original is returned unchanged.
     :param vo: The long VO name string.
     :returns: The short VO name string.
     """

rucio/core/volatile_replica.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,13 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
+from collections.abc import Iterable
 from datetime import datetime
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
 
-from sqlalchemy import and_, or_, exists, update, insert
-from sqlalchemy.orm.exc import NoResultFound
-from sqlalchemy.sql.expression import select
+from sqlalchemy import and_, delete, exists, insert, or_, select, true, update
+from sqlalchemy.exc import NoResultFound
 
 from rucio.common import exception
 from rucio.core.rse import get_rse_name
@@ -32,18 +30,23 @@ if TYPE_CHECKING:
 
 
 @transactional_session
-def add_volatile_replicas(rse_id, replicas, *, session: "Session"):
+def add_volatile_replicas(rse_id: str, replicas: Iterable[dict[str, Any]], *, session: "Session") -> None:
     """
     Bulk add volatile replicas.
 
     :param rse_id: the rse id.
-    :param replicas: the list of volatile replicas.
+    :param replicas: the iterable of volatile replicas.
     :param session: The database session in use.
-    :returns: True is successful.
     """
     # first check that the rse is a volatile one
     try:
-        session.query(models.RSE).filter_by(id=rse_id, volatile=True).one()
+        stmt = select(
+            models.RSE
+        ).where(
+            and_(models.RSE.id == rse_id,
+                 models.RSE.volatile == true())
+        )
+        session.execute(stmt).one()
     except NoResultFound:
         raise exception.UnsupportedOperation('No volatile rse found for %s !'
                                              % get_rse_name(rse_id=rse_id, session=session))
@@ -52,7 +55,7 @@ def add_volatile_replicas(rse_id, replicas, *, session: "Session"):
     for replica in replicas:
         file_clause.append(and_(models.DataIdentifier.scope == replica['scope'],
                                 models.DataIdentifier.name == replica['name'],
-                                ~exists(select(1).prefix_with("/*+ INDEX(REPLICAS REPLICAS_PK) */", dialect='oracle'))
+                                ~exists(select(1).prefix_with('/*+ INDEX(REPLICAS REPLICAS_PK) */', dialect='oracle'))
                                 .where(and_(models.RSEFileAssociation.scope == replica['scope'],
                                             models.RSEFileAssociation.name == replica['name'],
                                             models.RSEFileAssociation.rse_id == rse_id))))
@@ -61,20 +64,31 @@ def add_volatile_replicas(rse_id, replicas, *, session: "Session"):
                                    models.RSEFileAssociation.rse_id == rse_id))
     if replica_clause:
         now = datetime.utcnow()
-        stmt = update(models.RSEFileAssociation).\
-            prefix_with("/*+ index(REPLICAS REPLICAS_PK) */", dialect='oracle').\
-            where(or_(*replica_clause)).\
-            execution_options(synchronize_session=False).\
-            values(updated_at=now, tombstone=now)
+        stmt = update(
+            models.RSEFileAssociation
+        ).prefix_with(
+            '/*+ INDEX(REPLICAS REPLICAS_PK) */',
+            dialect='oracle'
+        ).where(
+            or_(*replica_clause)
+        ).execution_options(
+            synchronize_session=False
+        ).values({
+            models.RSEFileAssociation.updated_at: now,
+            models.RSEFileAssociation.tombstone: now
+        })
         session.execute(stmt)
 
     if file_clause:
-        file_query = session.query(models.DataIdentifier.scope,
-                                   models.DataIdentifier.name,
-                                   models.DataIdentifier.bytes,
-                                   models.DataIdentifier.md5,
-                                   models.DataIdentifier.adler32).\
-            filter(or_(*file_clause))
+        stmt = select(
+            models.DataIdentifier.scope,
+            models.DataIdentifier.name,
+            models.DataIdentifier.bytes,
+            models.DataIdentifier.md5,
+            models.DataIdentifier.adler32
+        ).filter(
+            or_(*file_clause)
+        )
 
         new_replicas = [
             {
@@ -88,25 +102,33 @@ def add_volatile_replicas(rse_id, replicas, *, session: "Session"):
                 'bytes': bytes_,
                 'md5': md5
             }
-            for scope, name, bytes_, md5, adler32 in file_query
+            for scope, name, bytes_, md5, adler32 in session.execute(stmt).all()
         ]
         if new_replicas:
-            session.execute(insert(models.RSEFileAssociation), new_replicas)
+            stmt = insert(
+                models.RSEFileAssociation
+            )
+            session.execute(stmt, new_replicas)
 
 
 @transactional_session
-def delete_volatile_replicas(rse_id, replicas, *, session: "Session"):
+def delete_volatile_replicas(rse_id: str, replicas: Iterable[dict[str, Any]], *, session: "Session") -> None:
     """
     Bulk delete volatile replicas.
 
     :param rse_id: the rse id.
-    :param replicas: the list of volatile replicas.
+    :param replicas: the iterable of volatile replicas.
     :param session: The database session in use.
-    :returns: True is successful.
     """
     # first check that the rse is a volatile one
     try:
-        session.query(models.RSE).filter_by(id=rse_id, volatile=True).one()
+        stmt = select(
+            models.RSE
+        ).where(
+            and_(models.RSE.id == rse_id,
+                 models.RSE.volatile == true())
+        )
+        session.execute(stmt).one()
     except NoResultFound:
         raise exception.UnsupportedOperation('No volatile rse found for %s !'
                                              % get_rse_name(rse_id=rse_id, session=session))
@@ -117,7 +139,12 @@ def delete_volatile_replicas(rse_id, replicas, *, session: "Session"):
                                models.RSEFileAssociation.name == replica['name']))
 
     if conditions:
-        session.query(models.RSEFileAssociation).\
-            filter(models.RSEFileAssociation.rse_id == rse_id).\
-            filter(or_(*conditions)).\
-            delete(synchronize_session=False)
+        stmt = delete(
+            models.RSEFileAssociation
+        ).where(
+            and_(models.RSEFileAssociation.rse_id == rse_id,
+                 or_(*conditions))
+        ).execution_options(
+            synchronize_session=False
+        )
+        session.execute(stmt)

rucio/daemons/__init__.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

rucio/daemons/abacus/__init__.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

rucio/daemons/abacus/account.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -26,8 +25,8 @@ import rucio.db.sqla.util
 from rucio.common import exception
 from rucio.common.logging import setup_logging
 from rucio.common.utils import get_thread_with_periodic_running_function
-from rucio.core.account_counter import get_updated_account_counters, update_account_counter, fill_account_counter_history_table
-from rucio.daemons.common import run_daemon
+from rucio.core.account_counter import fill_account_counter_history_table, get_updated_account_counters, update_account_counter
+from rucio.daemons.common import HeartbeatHandler, run_daemon
 
 if TYPE_CHECKING:
     from types import FrameType
@@ -37,7 +36,10 @@ graceful_stop = threading.Event()
 DAEMON_NAME = 'abacus-account'
 
 
-def account_update(once=False, sleep_time=10):
+def account_update(
+        once: bool = False,
+        sleep_time: int = 10
+) -> None:
     """
     Main loop to check and update the Account Counters.
     """
@@ -51,26 +53,29 @@ def account_update(once=False, sleep_time=10):
     )
 
 
-def run_once(heartbeat_handler, **_kwargs):
+def run_once(
+        heartbeat_handler: HeartbeatHandler,
+        **_kwargs
+) -> None:
     worker_number, total_workers, logger = heartbeat_handler.live()
 
     start = time.time()  # NOQA
-    account_rse_ids = get_updated_account_counters(total_workers=total_workers,
-                                                   worker_number=worker_number)
-    logger(logging.DEBUG, 'Index query time %f size=%d' % (time.time() - start, len(account_rse_ids)))
+    updated_account_counters = get_updated_account_counters(total_workers=total_workers,
+                                                            worker_number=worker_number)
+    logger(logging.DEBUG, 'Index query time %f size=%d' % (time.time() - start, len(updated_account_counters)))
 
     # If the list is empty, sent the worker to sleep
-    if not account_rse_ids:
+    if not updated_account_counters:
         logger(logging.INFO, 'did not get any work')
         return
 
-    for account_rse_id in account_rse_ids:
+    for account_counter in updated_account_counters:
         worker_number, total_workers, logger = heartbeat_handler.live()
         if graceful_stop.is_set():
             break
         start_time = time.time()
-        update_account_counter(account=account_rse_id[0], rse_id=account_rse_id[1])
-        logger(logging.DEBUG, 'update of account-rse counter "%s-%s" took %f' % (account_rse_id[0], account_rse_id[1], time.time() - start_time))
+        update_account_counter(account=account_counter['account'], rse_id=account_counter['rse_id'])
+        logger(logging.DEBUG, 'update of account-rse counter "%s-%s" took %f' % (account_counter['account'], account_counter['rse_id'], time.time() - start_time))
 
 
 def stop(signum: "Optional[int]" = None, frame: "Optional[FrameType]" = None) -> None:
@@ -81,7 +86,12 @@ def stop(signum: "Optional[int]" = None, frame: "Optional[FrameType]" = None) ->
     graceful_stop.set()
 
 
-def run(once=False, threads=1, fill_history_table=False, sleep_time=10):
+def run(
+        once: bool = False,
+        threads: int = 1,
+        fill_history_table: bool = False,
+        sleep_time: int = 10
+) -> None:
     """
     Starts up the Abacus-Account threads.
     """
@@ -95,12 +105,12 @@ def run(once=False, threads=1, fill_history_table=False, sleep_time=10):
         account_update(once)
     else:
         logging.info('main: starting threads')
-        threads = [threading.Thread(target=account_update, kwargs={'once': once, 'sleep_time': sleep_time}) for i in
-                   range(0, threads)]
+        thread_list = [threading.Thread(target=account_update, kwargs={'once': once, 'sleep_time': sleep_time}) for i in
+                       range(0, threads)]
         if fill_history_table:
-            threads.append(get_thread_with_periodic_running_function(3600, fill_account_counter_history_table, graceful_stop))
-        [t.start() for t in threads]
+            thread_list.append(get_thread_with_periodic_running_function(3600, fill_account_counter_history_table, graceful_stop))
+        [t.start() for t in thread_list]
         logging.info('main: waiting for interrupts')
         # Interruptible joins require a timeout.
-        while threads[0].is_alive():
-            [t.join(timeout=3.14) for t in threads]
+        while thread_list[0].is_alive():
+            [t.join(timeout=3.14) for t in thread_list]

rucio/daemons/abacus/collection_replica.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -26,7 +25,7 @@ import rucio.db.sqla.util
 from rucio.common import exception
 from rucio.common.logging import setup_logging
 from rucio.core.replica import get_cleaned_updated_collection_replicas, update_collection_replica
-from rucio.daemons.common import run_daemon
+from rucio.daemons.common import HeartbeatHandler, run_daemon
 
 if TYPE_CHECKING:
     from types import FrameType
@@ -36,7 +35,11 @@ graceful_stop = threading.Event()
 DAEMON_NAME = 'abacus-collection-replica'
 
 
-def collection_replica_update(once=False, limit=1000, sleep_time=10):
+def collection_replica_update(
+        once: bool = False,
+        limit: int = 1000,
+        sleep_time: int = 10
+) -> None:
     """
     Main loop to check and update the collection replicas.
     """
@@ -53,7 +56,11 @@ def collection_replica_update(once=False, limit=1000, sleep_time=10):
     )
 
 
-def run_once(heartbeat_handler, limit, **_kwargs):
+def run_once(
+        heartbeat_handler: HeartbeatHandler,
+        limit: int,
+        **_kwargs
+) -> bool:
     worker_number, total_workers, logger = heartbeat_handler.live()
     # Select a bunch of collection replicas for to update for this worker
     start = time.time()  # NOQA
@@ -90,7 +97,11 @@ def stop(signum: "Optional[int]" = None, frame: "Optional[FrameType]" = None) ->
     graceful_stop.set()
 
 
-def run(once=False, threads=1, sleep_time=10, limit=1000):
+def run(
+        once: bool = False,
+        threads: int = 1,
+        sleep_time: int = 10,
+        limit: int = 1000):
     """
     Starts up the Abacus-Collection-Replica threads.
     """
@@ -104,10 +115,10 @@ def run(once=False, threads=1, sleep_time=10, limit=1000):
         collection_replica_update(once)
     else:
         logging.info('main: starting threads')
-        threads = [threading.Thread(target=collection_replica_update, kwargs={'once': once, 'sleep_time': sleep_time, 'limit': limit})
-                   for _ in range(0, threads)]
-        [t.start() for t in threads]
+        thread_list = [threading.Thread(target=collection_replica_update, kwargs={'once': once, 'sleep_time': sleep_time, 'limit': limit})
+                       for _ in range(0, threads)]
+        [t.start() for t in thread_list]
         logging.info('main: waiting for interrupts')
         # Interruptible joins require a timeout.
-        while threads[0].is_alive():
-            [t.join(timeout=3.14) for t in threads]
+        while thread_list[0].is_alive():
+            [t.join(timeout=3.14) for t in thread_list]

rucio/daemons/abacus/rse.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -26,8 +25,8 @@ import rucio.db.sqla.util
 from rucio.common import exception
 from rucio.common.logging import setup_logging
 from rucio.common.utils import get_thread_with_periodic_running_function
-from rucio.core.rse_counter import get_updated_rse_counters, update_rse_counter, fill_rse_counter_history_table
-from rucio.daemons.common import run_daemon
+from rucio.core.rse_counter import fill_rse_counter_history_table, get_updated_rse_counters, update_rse_counter
+from rucio.daemons.common import HeartbeatHandler, run_daemon
 
 if TYPE_CHECKING:
     from types import FrameType
@@ -37,7 +36,10 @@ graceful_stop = threading.Event()
 DAEMON_NAME = 'abacus-rse'
 
 
-def rse_update(once=False, sleep_time=10):
+def rse_update(
+        once: bool = False,
+        sleep_time: int = 10
+) -> None:
     """
     Main loop to check and update the RSE Counters.
     """
@@ -51,7 +53,10 @@ def rse_update(once=False, sleep_time=10):
     )
 
 
-def run_once(heartbeat_handler, **_kwargs):
+def run_once(
+        heartbeat_handler: HeartbeatHandler,
+        **_kwargs
+) -> None:
     worker_number, total_workers, logger = heartbeat_handler.live()
 
     # Select a bunch of rses for to update for this worker
@@ -82,7 +87,12 @@ def stop(signum: "Optional[int]" = None, frame: "Optional[FrameType]" = None) ->
     graceful_stop.set()
 
 
-def run(once=False, threads=1, fill_history_table=False, sleep_time=10):
+def run(
+        once: bool = False,
+        threads: int = 1,
+        fill_history_table: bool = False,
+        sleep_time: int = 10
+) -> None:
     """
     Starts up the Abacus-RSE threads.
     """
@@ -96,12 +106,12 @@ def run(once=False, threads=1, fill_history_table=False, sleep_time=10):
         rse_update(once)
     else:
         logging.info('main: starting threads')
-        threads = [threading.Thread(target=rse_update, kwargs={'once': once, 'sleep_time': sleep_time}) for i in
-                   range(0, threads)]
+        thread_list = [threading.Thread(target=rse_update, kwargs={'once': once, 'sleep_time': sleep_time}) for i in
+                       range(0, threads)]
         if fill_history_table:
-            threads.append(get_thread_with_periodic_running_function(3600, fill_rse_counter_history_table, graceful_stop))
-        [t.start() for t in threads]
+            thread_list.append(get_thread_with_periodic_running_function(3600, fill_rse_counter_history_table, graceful_stop))
+        [t.start() for t in thread_list]
         logging.info('main: waiting for interrupts')
         # Interruptible joins require a timeout.
-        while threads[0].is_alive():
-            [t.join(timeout=3.14) for t in threads]
+        while thread_list[0].is_alive():
+            [t.join(timeout=3.14) for t in thread_list]

rucio/daemons/atropos/__init__.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

rucio/daemons/atropos/atropos.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -110,7 +109,7 @@ def run_once(
             lifetime_exceptions[key] = excep['expires_at']
     logger(logging.DEBUG, '%d active exceptions', len(lifetime_exceptions))
 
-    rand = random.Random(worker_number)
+    rand = random.Random(worker_number)   # noqa: S311
 
     try:
         rules = get_rules_beyond_eol(date_check, worker_number, total_workers, session=None)