rucio 32.8.6__py3-none-any.whl → 35.8.0__py3-none-any.whl

rucio/core/scope.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,11 +14,13 @@
 
 from re import match
 from traceback import format_exc
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any, Optional
 
+from sqlalchemy import and_, select
 from sqlalchemy.exc import IntegrityError
 
 from rucio.common.exception import AccountNotFound, Duplicate, RucioException, VONotFound
+from rucio.common.types import InternalScope
 from rucio.core.vo import vo_exists
 from rucio.db.sqla import models
 from rucio.db.sqla.constants import AccountStatus, ScopeStatus
@@ -41,8 +42,13 @@ def add_scope(scope, account, *, session: "Session"):
     if not vo_exists(vo=scope.vo, session=session):
         raise VONotFound('VO {} not found'.format(scope.vo))
 
-    result = session.query(models.Account).filter_by(account=account, status=AccountStatus.ACTIVE).first()
-    if result is None:
+    stmt = select(
+        models.Account
+    ).where(
+        and_(models.Account.account == account,
+             models.Account.status == AccountStatus.ACTIVE)
+    )
+    if session.execute(stmt).first() is None:
         raise AccountNotFound('Account ID \'%s\' does not exist' % account)
 
     new_scope = models.Scope(scope=scope, account=account, status=ScopeStatus.OPEN)
@@ -80,7 +86,7 @@ def bulk_add_scopes(scopes, account, skipExisting=False, *, session: "Session"):
 
 
 @read_session
-def list_scopes(filter_={}, *, session: "Session"):
+def list_scopes(filter_: Optional[dict[str, Any]] = None, *, session: "Session") -> list[InternalScope]:
     """
     Lists all scopes.
     :param filter_: Dictionary of attributes by which the input data should be filtered
@@ -88,19 +94,25 @@ def list_scopes(filter_={}, *, session: "Session"):
 
     :returns: A list containing all scopes.
     """
-    scope_list = []
-    query = session.query(models.Scope).filter(models.Scope.status != ScopeStatus.DELETED)
+    filter_ = filter_ or {}
+    stmt = select(
+        models.Scope.scope
+    ).where(
+        models.Scope.status != ScopeStatus.DELETED
+    )
     for filter_type in filter_:
         if filter_type == 'scope':
             if '*' in filter_['scope'].internal:
                 scope_str = filter_['scope'].internal.replace('*', '%')
-                query = query.filter(models.Scope.scope.like(scope_str))
+                stmt = stmt.where(
+                    models.Scope.scope.like(scope_str)
+                )
             else:
-                query = query.filter_by(scope=filter_['scope'])
+                stmt = stmt.where(
+                    models.Scope.scope == filter_['scope']
+                )
 
-    for s in query:
-        scope_list.append(s.scope)
-    return scope_list
+    return list(session.execute(stmt).scalars().all())
 
 
 @read_session
@@ -113,17 +125,22 @@ def get_scopes(account, *, session: "Session"):
     :returns: a list of all scope names for this account.
     """
 
-    result = session.query(models.Account).filter_by(account=account).first()
+    stmt = select(
+        models.Account
+    ).where(
+        models.Account.account == account
+    )
 
-    if result is None:
+    if session.execute(stmt).first() is None:
         raise AccountNotFound('Account ID \'%s\' does not exist' % account)
 
-    scope_list = []
-
-    for s in session.query(models.Scope).filter_by(account=account).filter(models.Scope.status != ScopeStatus.DELETED):
-        scope_list.append(s.scope)
-
-    return scope_list
+    stmt = select(
+        models.Scope.scope
+    ).where(
+        and_(models.Scope.account == account,
+             models.Scope.status != ScopeStatus.DELETED)
+    )
+    return session.execute(stmt).scalars().all()
 
 
 @read_session
@@ -136,7 +153,12 @@ def check_scope(scope_to_check, *, session: "Session"):
     :returns: True or false
     """
 
-    return True if session.query(models.Scope).filter_by(scope=scope_to_check).first() else False
+    stmt = select(
+        models.Scope
+    ).where(
+        models.Scope.scope == scope_to_check
+    )
+    return bool(session.execute(stmt).scalar())
 
 
 @read_session
@@ -149,4 +171,10 @@ def is_scope_owner(scope, account, *, session: "Session"):
 
     :returns: True or false
     """
-    return True if session.query(models.Scope).filter_by(scope=scope, account=account).first() else False
+    stmt = select(
+        models.Scope
+    ).where(
+        and_(models.Scope.scope == scope,
+             models.Scope.account == account)
+    )
+    return bool(session.execute(stmt).scalar())

rucio/core/subscription.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -18,23 +17,23 @@ import logging
 import re
 from configparser import NoOptionError, NoSectionError
 from json import dumps
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any, Optional
 
-from sqlalchemy import func
-from sqlalchemy.exc import IntegrityError, StatementError
+from sqlalchemy import and_, delete, func, select
+from sqlalchemy.exc import IntegrityError, NoResultFound, StatementError
 from sqlalchemy.orm import aliased
-from sqlalchemy.orm.exc import NoResultFound
 
-from rucio.common.config import config_get
-from rucio.common.exception import SubscriptionNotFound, SubscriptionDuplicate, RucioException
+from rucio.common.config import config_get_bool
+from rucio.common.exception import RucioException, SubscriptionDuplicate, SubscriptionNotFound
 from rucio.db.sqla import models
-from rucio.db.sqla.constants import SubscriptionState
-from rucio.db.sqla.session import transactional_session, stream_session, read_session
+from rucio.db.sqla.constants import RuleState, SubscriptionState
+from rucio.db.sqla.session import read_session, stream_session, transactional_session
 
 if TYPE_CHECKING:
     from collections.abc import Callable, Iterator
-    from typing import Any, Optional
+
     from sqlalchemy.orm import Session
+
     from rucio.common.types import InternalAccount
     LoggerFunction = Callable[..., Any]
     SubscriptionType = dict
@@ -46,10 +45,10 @@ def add_subscription(name: str,
                      filter_: str,
                      replication_rules: str,
                      comments: str,
-                     lifetime: "Optional[int]" = None,
-                     retroactive: "Optional[bool]" = False,
-                     dry_run: "Optional[bool]" = False,
-                     priority: "Optional[int]" = 3,
+                     lifetime: Optional[int] = None,
+                     retroactive: Optional[bool] = False,
+                     dry_run: Optional[bool] = False,
+                     priority: Optional[int] = 3,
                      *, session: "Session") -> str:
     """
     Adds a new subscription which will be verified against every new added file and dataset
@@ -78,7 +77,7 @@ def add_subscription(name: str,
     :returns:                  The subscriptionid
     """
     try:
-        keep_history = config_get('subscriptions', 'keep_history')
+        keep_history = config_get_bool('subscriptions', 'keep_history')
     except (NoOptionError, NoSectionError, RuntimeError):
         keep_history = False
 
@@ -109,7 +108,9 @@ def add_subscription(name: str,
                                                    lifetime=new_subscription.lifetime,
                                                    retroactive=new_subscription.retroactive,
                                                    policyid=new_subscription.policyid,
-                                                   comments=new_subscription.comments)
+                                                   comments=new_subscription.comments,
+                                                   created_at=datetime.datetime.utcnow(),
+                                                   updated_at=datetime.datetime.utcnow())
     try:
         new_subscription.save(session=session)
         if keep_history:
@@ -146,10 +147,10 @@ def update_subscription(name: str,
     :raises: SubscriptionNotFound if subscription is not found
     """
     try:
-        keep_history = config_get('subscriptions', 'keep_history')
+        keep_history = config_get_bool('subscriptions', 'keep_history')
     except (NoOptionError, NoSectionError, RuntimeError):
         keep_history = False
-    values = {'state': SubscriptionState.UPDATED}
+    values: dict[str, Any] = {'state': SubscriptionState.UPDATED}
     if 'filter' in metadata and metadata['filter']:
         values['filter'] = dumps(metadata['filter'])
     if 'replication_rules' in metadata and metadata['replication_rules']:
@@ -172,7 +173,13 @@ def update_subscription(name: str,
 
     SubscriptionHistory = models.SubscriptionHistory
     try:
-        subscription = session.query(models.Subscription).filter_by(account=account, name=name).one()
+        stmt = select(
+            models.Subscription
+        ).where(
+            and_(models.Subscription.name == name,
+                 models.Subscription.account == account)
+        )
+        subscription = session.execute(stmt).scalar_one()
 
         # To avoid update in the subscription history table whenever last processed field is changed
         current_subscription_state = subscription.to_dict()
@@ -196,7 +203,7 @@ def update_subscription(name: str,
                                                        comments=subscription.comments,
                                                        last_processed=subscription.last_processed,
                                                        expired_at=subscription.expired_at,
-                                                       updated_at=subscription.updated_at,
+                                                       updated_at=datetime.datetime.utcnow(),
                                                        created_at=subscription.created_at)
             subscription_history.save(session=session)
     except NoResultFound:
@@ -204,9 +211,9 @@ def update_subscription(name: str,
 
 
 @stream_session
-def list_subscriptions(name: "Optional[str]" = None,
+def list_subscriptions(name: Optional[str] = None,
                        account: "Optional[InternalAccount]" = None,
-                       state: "Optional[SubscriptionState]" = None,
+                       state: Optional[SubscriptionState] = None,
                        *, session: "Session",
                        logger: "LoggerFunction" = logging.log) -> "Iterator[SubscriptionType]":
     """
@@ -224,23 +231,33 @@ def list_subscriptions(name: "Optional[str]" = None,
     :rtype:                    Dict
     :raises:                   exception.NotFound if subscription is not found
     """
-    query = session.query(models.Subscription)
+    stmt = select(
+        models.Subscription
+    )
     try:
         if name:
-            query = query.filter_by(name=name)
+            stmt = stmt.where(
+                models.Subscription.name == name
+            )
         if account:
-            if '*' in account.internal:
+            if account.internal is not None and '*' in account.internal:
                 account_str = account.internal.replace('*', '%')
-                query = query.filter(models.Subscription.account.like(account_str))
+                stmt = stmt.where(
+                    models.Subscription.account.like(account_str)
+                )
             else:
-                query = query.filter_by(account=account)
+                stmt = stmt.where(
+                    models.Subscription.account == account
+                )
         if state:
-            query = query.filter_by(state=state)
+            stmt = stmt.where(
+                models.Subscription.state == state
+            )
     except IntegrityError as error:
         logger(logging.ERROR, str(error))
         raise RucioException(error.args)
     found = False
-    for row in query:
+    for row in session.execute(stmt).scalars().all():
         found = True
         yield row.to_dict()
     if not found:
@@ -255,11 +272,22 @@ def delete_subscription(subscription_id: str, *, session: "Session") -> None:
     :param subscription_id: Subscription identifier
     :type subscription_id:  String
     """
-    session.query(models.Subscription).filter_by(id=subscription_id).delete()
+    stmt = delete(
+        models.Subscription
+    ).where(
+        models.Subscription.id == subscription_id
+    )
+    session.execute(stmt)
 
 
 @stream_session
-def list_subscription_rule_states(name=None, account=None, *, session: "Session", logger=logging.log):
+def list_subscription_rule_states(
+    name: Optional[str] = None,
+    account: Optional["InternalAccount"] = None,
+    *,
+    session: "Session",
+    logger: "LoggerFunction" = logging.log
+) -> "Iterator[tuple[InternalAccount, str, RuleState, int]]":
     """Returns a list of with the number of rules per state for a subscription.
 
     :param name:               Name of the subscription
@@ -271,33 +299,49 @@ def list_subscription_rule_states(name=None, account=None, *, session: "Session"
     subscription = aliased(models.Subscription)
     rule = aliased(models.ReplicationRule)
     # count needs a label to allow conversion to dict (label name can be changed)
-    query = session.query(
-        subscription.account, subscription.name, rule.state, func.count().label('count')).join(
-        rule, subscription.id == rule.subscription_id)
+    stmt = select(
+        subscription.account,
+        subscription.name,
+        rule.state,
+        func.count().label('count')
+    ).join(
+        rule,
+        subscription.id == rule.subscription_id
+    )
 
     try:
         if name:
-            query = query.filter(subscription.name == name)
+            stmt = stmt.where(
+                subscription.name == name
+            )
 
         if account:
-            if '*' in account.internal:
+            if account.internal and '*' in account.internal:
                 account_str = account.internal.replace('*', '%')
-                query = query.filter(subscription.account.like(account_str))
+                stmt = stmt.where(
+                    subscription.account.like(account_str)
+                )
             else:
-                query = query.filter(subscription.account == account)
+                stmt = stmt.where(
+                    subscription.account == account
+                )
 
     except IntegrityError as error:
         logger(logging.ERROR, str(error))
         raise RucioException(error.args)
 
-    query = query.group_by(subscription.account, subscription.name, rule.state)
+    stmt = stmt.group_by(
+        subscription.account,
+        subscription.name,
+        rule.state
+    )
 
-    for row in query:
-        yield row
+    for row in session.execute(stmt).all():
+        yield row._tuple()
 
 
 @read_session
-def get_subscription_by_id(subscription_id, *, session: "Session"):
+def get_subscription_by_id(subscription_id: str, *, session: "Session") -> "SubscriptionType":
     """
     Get a specific subscription by id.
 
@@ -308,8 +352,12 @@ def get_subscription_by_id(subscription_id, *, session: "Session"):
     """
 
     try:
-        subscription = session.query(models.Subscription).filter_by(id=subscription_id).one()
-        return subscription.to_dict()
+        stmt = select(
+            models.Subscription
+        ).where(
+            models.Subscription.id == subscription_id
+        )
+        return session.execute(stmt).scalar_one().to_dict()
     except NoResultFound:
         raise SubscriptionNotFound('No subscription with the id %s found' % (subscription_id))
     except StatementError:

rucio/core/topology.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import copy
 import datetime
 import itertools
 import logging
@@ -19,12 +19,12 @@ import threading
 import weakref
 from collections.abc import Callable, Iterable, Iterator
 from decimal import Decimal
-from typing import TYPE_CHECKING, cast, Any, Generic, Optional, TypeVar, Union
+from typing import TYPE_CHECKING, Any, Generic, Optional, TypeVar, Union, cast
 
 from sqlalchemy import and_, select
 
-from rucio.common.config import config_get_int, config_get
-from rucio.common.exception import NoDistance, RSEProtocolNotSupported, InvalidRSEExpression
+from rucio.common.config import config_get, config_get_int
+from rucio.common.exception import InvalidRSEExpression, NoDistance, RSEProtocolNotSupported
 from rucio.common.utils import PriorityQueue
 from rucio.core.rse import RseCollection, RseData
 from rucio.core.rse_expression_parser import parse_expression
@@ -32,15 +32,18 @@ from rucio.db.sqla import models
 from rucio.db.sqla.session import read_session, transactional_session
 from rucio.rse import rsemanager as rsemgr
 
-LoggerFunction = Callable[..., Any]
 _Number = Union[int, Decimal]
 TN = TypeVar("TN", bound="Node")
 TE = TypeVar("TE", bound="Edge")
+ExpiringObjectCacheNewObject = TypeVar("ExpiringObjectCacheNewObject")
 
 if TYPE_CHECKING:
+    from typing import Protocol
+
     from sqlalchemy.orm import Session
+    from typing_extensions import Self
 
-    from typing import Protocol
+    from rucio.common.types import HopDict, LoggerFunction
 
     class _StateProvider(Protocol):
         @property
@@ -81,11 +84,11 @@ class Edge(Generic[TN]):
 
         self.add_to_nodes()
 
-    def add_to_nodes(self):
+    def add_to_nodes(self) -> None:
         self.src_node.out_edges[self.dst_node] = self
         self.dst_node.in_edges[self.src_node] = self
 
-    def remove_from_nodes(self):
+    def remove_from_nodes(self) -> None:
         self.src_node.out_edges.pop(self.dst_node, None)
         self.dst_node.in_edges.pop(self.src_node, None)
 
@@ -105,12 +108,12 @@ class Edge(Generic[TN]):
             raise ReferenceError("weak reference returned None")
         return node
 
-    def __eq__(self, other):
+    def __eq__(self, other: object) -> bool:
         if not isinstance(other, self.__class__):
             return False
         return self._src_node == other._src_node and self._dst_node == other._dst_node
 
-    def __str__(self):
+    def __str__(self) -> str:
         return f'{self._src_node}-->{self._dst_node}'
 
 
@@ -127,7 +130,7 @@ class Topology(RseCollection, Generic[TN, TE]):
     ):
         super().__init__(rse_ids=rse_ids, rse_data_cls=node_cls)
         self._edge_cls = edge_cls
-        self._edges = {}
+        self._edges: dict[tuple[TN, TN], TE] = {}
         self._edges_loaded = False
         self._multihop_nodes = set()
         self._hop_penalty = DEFAULT_HOP_PENALTY
@@ -135,6 +138,36 @@ class Topology(RseCollection, Generic[TN, TE]):
 
         self._lock = threading.RLock()
 
+    @transactional_session
+    def ensure_loaded(
+            self,
+            rse_ids: "Optional[Iterable[str]]" = None,
+            load_name: bool = False,
+            load_columns: bool = False,
+            load_attributes: bool = False,
+            load_info: bool = False,
+            load_usage: bool = False,
+            load_limits: bool = False,
+            include_deleted: bool = False,
+            *,
+            session: "Session",
+    ) -> None:
+
+        if not rse_ids:
+            with self._lock:
+                rse_ids = list(self.rse_id_to_data_map)
+        super().ensure_loaded(
+            rse_ids=rse_ids,
+            load_name=load_name,
+            load_columns=load_columns,
+            load_attributes=load_attributes,
+            load_info=load_info,
+            load_usage=load_usage,
+            load_limits=load_limits,
+            include_deleted=include_deleted,
+            session=session,
+        )
+
     def get_or_create(self, rse_id: str) -> "TN":
         rse_data = self.rse_id_to_data_map.get(rse_id)
         if rse_data is None:
@@ -146,6 +179,11 @@ class Topology(RseCollection, Generic[TN, TE]):
                     self._edges_loaded = False
         return rse_data
 
+    @property
+    def edges(self) -> dict[tuple[TN, TN], TE]:
+        with self._lock:
+            return copy.copy(self._edges)
+
     def edge(self, src_node: TN, dst_node: TN) -> "Optional[TE]":
         return self._edges.get((src_node, dst_node))
 
@@ -158,7 +196,7 @@ class Topology(RseCollection, Generic[TN, TE]):
                     self._edges[src_node, dst_node] = edge = self._edge_cls(src_node, dst_node)
         return edge
 
-    def delete_edge(self, src_node: TN, dst_node: TN):
+    def delete_edge(self, src_node: TN, dst_node: TN) -> None:
         with self._lock:
             edge = self._edges[src_node, dst_node]
             edge.remove_from_nodes()
@@ -168,11 +206,11 @@ class Topology(RseCollection, Generic[TN, TE]):
         return True if self._multihop_nodes else False
 
     @read_session
-    def configure_multihop(self, multihop_rse_ids: Optional[set[str]] = None, *, session: "Session", logger: LoggerFunction = logging.log):
+    def configure_multihop(self, multihop_rse_ids: Optional[set[str]] = None, *, session: "Session", logger: "LoggerFunction" = logging.log) -> "Self":
         with self._lock:
             return self._configure_multihop(multihop_rse_ids=multihop_rse_ids, session=session, logger=logger)
 
-    def _configure_multihop(self, multihop_rse_ids: Optional[set[str]] = None, *, session: "Session", logger: LoggerFunction = logging.log):
+    def _configure_multihop(self, multihop_rse_ids: Optional[set[str]] = None, *, session: "Session", logger: "LoggerFunction" = logging.log) -> "Self":
 
         if multihop_rse_ids is None:
             multihop_rse_expression = config_get('transfers', 'multihop_rse_expression', default='available_for_multihop=true', expiration_time=600, session=session)
@@ -201,7 +239,7 @@ class Topology(RseCollection, Generic[TN, TE]):
         return self
 
     @read_session
-    def ensure_edges_loaded(self, *, session: "Session"):
+    def ensure_edges_loaded(self, *, session: "Session") -> None:
         """
         Ensure that all edges are loaded for the (sub-)set of nodes known by this topology object
         """
@@ -211,7 +249,7 @@ class Topology(RseCollection, Generic[TN, TE]):
         with self._lock:
             return self._ensure_edges_loaded(session=session)
 
-    def _ensure_edges_loaded(self, *, session: "Session"):
+    def _ensure_edges_loaded(self, *, session: "Session") -> None:
         stmt = select(
             models.Distance
         ).where(
@@ -246,7 +284,7 @@ class Topology(RseCollection, Generic[TN, TE]):
     @read_session
     def search_shortest_paths(
             self,
-            src_nodes: list[TN],
+            src_nodes: Iterable[TN],
             dst_node: TN,
             operation_src: str,
             operation_dest: str,
@@ -320,7 +358,7 @@ class Topology(RseCollection, Generic[TN, TE]):
                                                                                  node_state_provider=_NodeStateProvider,
                                                                                  edge_state_provider=_EdgeStateProvider):
             nh_node = edge_to_next_hop.dst_node
-            edge_state = cast(_EdgeStateProvider, edge_state)
+            edge_state = cast("_EdgeStateProvider", edge_state)
             hop = {
                 'source_rse': node,
                 'dest_rse': nh_node,
@@ -386,20 +424,24 @@ class Topology(RseCollection, Generic[TN, TE]):
                             priority_q[adjacent_node] = new_adjacent_dist
 
 
-class ExpiringObjectCache:
+class ExpiringObjectCache(Generic[ExpiringObjectCacheNewObject]):
     """
     Thread-safe container which builds and object with the function passed in parameter and
     caches it for the TTL duration.
     """
 
-    def __init__(self, ttl, new_obj_fnc):
+    def __init__(
+            self,
+            ttl: int,
+            new_obj_fnc: Callable[[], ExpiringObjectCacheNewObject]
+    ):
         self._lock = threading.Lock()
-        self._object = None
-        self._creation_time = None
+        self._object: Optional[ExpiringObjectCacheNewObject] = None
+        self._creation_time: Optional[datetime.datetime] = None
         self._new_obj_fnc = new_obj_fnc
         self._ttl = ttl
 
-    def get(self, logger=logging.log):
+    def get(self, logger: "LoggerFunction" = logging.log) -> ExpiringObjectCacheNewObject:
         with self._lock:
             if not self._object \
                     or not self._creation_time \
@@ -417,7 +459,7 @@ def get_hops(
         multihop_rse_ids: Optional[set[str]] = None,
         limit_dest_schemes: Optional[list[str]] = None,
         *, session: "Session",
-):
+) -> list["HopDict"]:
     """
     Get a list of hops needed to transfer date from source_rse_id to dest_rse_id.
     Ideally, the list will only include one item (dest_rse_id) since no hops are needed.