rucio 35.7.0__py3-none-any.whl → 37.0.0__py3-none-any.whl

rucio/core/replica_sorter.py

@@ -21,7 +21,7 @@ from datetime import datetime, timedelta
 from math import asin, cos, radians, sin, sqrt
 from pathlib import Path
 from tempfile import TemporaryDirectory, TemporaryFile
-from typing import Optional, Union
+from typing import IO, TYPE_CHECKING, Any, Optional, Union
 from urllib.parse import urlparse
 
 import geoip2.database
@@ -29,20 +29,29 @@ import requests
 from dogpile.cache.api import NO_VALUE
 
 from rucio.common import utils
-from rucio.common.cache import make_region_memcached
+from rucio.common.cache import MemcacheRegion
 from rucio.common.config import config_get, config_get_bool, config_get_int
 from rucio.common.constants import SORTING_ALGORITHMS
 from rucio.common.exception import InvalidRSEExpression, SortingAlgorithmNotSupported
 from rucio.core.rse_expression_parser import parse_expression
 
-REGION = make_region_memcached(expiration_time=900, function_key_generator=utils.my_key_generator)
+if TYPE_CHECKING:
+    from _typeshed import StrPath
+
+    from rucio.common.types import IPDict, ReplicaDict
+
+REGION = MemcacheRegion(expiration_time=900, function_key_generator=utils.my_key_generator)
 
 # This product uses GeoLite data created by MaxMind,
 # available from <a href="http://www.maxmind.com">http://www.maxmind.com</a>
 GEOIP_DB_EDITION = 'GeoLite2-City'
 
 
-def extract_file_from_tar_gz(archive_file_obj, file_name, destination):
+def extract_file_from_tar_gz(
+        archive_file_obj: IO,
+        file_name: str,
+        destination: 'StrPath'
+) -> None:
     """
     Extract one file from the archive and put it at the destination
 
@@ -58,7 +67,7 @@ def extract_file_from_tar_gz(archive_file_obj, file_name, destination):
                     shutil.move(tmp_dir / entry.name, destination)
 
 
-def __download_geoip_db(destination):
+def __download_geoip_db(destination: 'StrPath') -> None:
     edition_id = GEOIP_DB_EDITION
     download_url = config_get('core', 'geoip_download_url', raise_exception=False, default=None)
     verify_tls = config_get_bool('core', 'geoip_download_verify_tls', raise_exception=False, default=True)
@@ -82,7 +91,7 @@ def __download_geoip_db(destination):
                                                                                      result.text))
 
 
-def __geoip_db():
+def __geoip_db() -> geoip2.database.Reader:
     db_path = Path(f'/tmp/{GEOIP_DB_EDITION}.mmdb')
     db_expire_delay = timedelta(days=config_get_int('core', 'geoip_expire_delay', raise_exception=False, default=30))
 
@@ -100,10 +109,13 @@ def __geoip_db():
     return geoip2.database.Reader(str(db_path))
 
 
-def __get_lat_long(se, gi):
+def __get_lat_long(
+        se: str,
+        gi: geoip2.database.Reader
+) -> tuple[Optional[float], Optional[float]]:
     """
     Get the latitude and longitude on one host using the GeoLite DB
-    :param se  : A hostname or IP.
+    :param se : A hostname or IP.
     :param gi : A Reader object (geoip2 API).
     """
     try:
@@ -116,7 +128,11 @@ def __get_lat_long(se, gi):
     return None, None
 
 
-def __get_distance(se1, client_location, ignore_error):
+def __get_distance(
+        se1: str,
+        client_location: 'IPDict',
+        ignore_error: bool
+) -> float:
     """
     Get the distance between 2 host using the GeoLite DB
     :param se1 : A first hostname or IP.
@@ -133,14 +149,15 @@ def __get_distance(se1, client_location, ignore_error):
             gi = __geoip_db()
 
             lat1, long1 = __get_lat_long(se1, gi)
-            if client_location.get('latitude') and client_location.get('longitude'):
-                lat2 = client_location['latitude']
-                long2 = client_location['longitude']
-            else:
+
+            lat2 = client_location.get('latitude')
+            long2 = client_location.get('longitude')
+
+            if not (lat2 and long2) and client_location['ip'] is not None:
                 lat2, long2 = __get_lat_long(client_location['ip'], gi)
 
             if lat1 and lat2:
-                long1, lat1, long2, lat2 = map(radians, [long1, lat1, long2, lat2])
+                long1, lat1, long2, lat2 = map(radians, [long1, lat1, long2, lat2])  # type: ignore (lat2 and long2 might be None)
                 dlon = long2 - long1
                 dlat = lat2 - lat1
                 cache_val = 6378 * 2 * asin(sqrt(sin(dlat / 2)**2 + cos(lat1) * cos(lat2) * sin(dlon / 2)**2))
@@ -152,7 +169,7 @@ def __get_distance(se1, client_location, ignore_error):
         # One host is on the Moon
         cache_val = 360000
         REGION.set(cache_key, cache_val)
-    return cache_val
+    return cache_val  # type: ignore (cache_val should not be None)
 
 
 def __download_custom_distance_table() -> None:
@@ -206,7 +223,7 @@ def __download_custom_distance_table() -> None:
             REGION.set(cache_key, distance)
 
 
-def __get_distance_custom(rse: Union[tuple, str], client_location: dict) -> float:
+def __get_distance_custom(rse: Union[tuple, str], client_location: 'IPDict') -> float:
     """
     Return the distance from a client to a RSE by looking up in custom distance table
     :param rse: RSE name, or tuple containing replica information with RSE name third
@@ -228,7 +245,11 @@ def __get_distance_custom(rse: Union[tuple, str], client_location: dict) -> floa
     return cache_val
 
 
-def site_selector(replicas, site, vo):
+def site_selector(
+        replicas: dict[str, 'ReplicaDict'],
+        site: str,
+        vo: str
+) -> list[str]:
     """
     Return a list of replicas located on one site.
     :param replicas : A dict with RSEs as values and replicas as keys (URIs).
@@ -249,7 +270,11 @@ def site_selector(replicas, site, vo):
     return result
 
 
-def sort_replicas(dictreplica: dict, client_location: dict, selection: Optional[str] = None) -> list:
+def sort_replicas(
+        dictreplica: dict[str, Any],
+        client_location: 'IPDict',
+        selection: Optional[str] = None
+) -> list[str]:
     """
     General sorting method for a dictionary of replicas. Returns the List of replicas.
 
@@ -279,19 +304,13 @@ def sort_replicas(dictreplica: dict, client_location: dict, selection: Optional[
         replicas = sort_geoip(dictreplica, client_location, ignore_error=config_get_bool('core', 'geoip_ignore_error', raise_exception=False, default=True))
     elif selection == 'custom_table':
         replicas = sort_custom(dictreplica, client_location)
-    elif selection == 'closeness':
-        replicas = sort_closeness(dictreplica, client_location)
-    elif selection == 'dynamic':
-        replicas = sort_dynamic(dictreplica, client_location)
-    elif selection == 'ranking':
-        replicas = sort_ranking(dictreplica, client_location)
     elif selection == 'random':
         replicas = sort_random(dictreplica)
 
     return replicas
 
 
-def sort_random(dictreplica: dict) -> list:
+def sort_random(dictreplica: dict[str, Any]) -> list[str]:
     """
     Return a list of replicas sorted randomly.
     :param dictreplica: A dict with replicas as keys (URIs).
@@ -302,7 +321,11 @@ def sort_random(dictreplica: dict) -> list:
     return list_replicas
 
 
-def sort_geoip(dictreplica: dict, client_location: dict, ignore_error: bool = False) -> list:
+def sort_geoip(
+        dictreplica: dict[str, Any],
+        client_location: 'IPDict',
+        ignore_error: bool = False
+) -> list[str]:
     """
     Return a list of replicas sorted by geographical distance to the client IP.
     :param dictreplica: A dict with replicas as keys (URIs).
@@ -310,19 +333,22 @@ def sort_geoip(dictreplica: dict, client_location: dict, ignore_error: bool = Fa
     :param ignore_error: Ignore exception when the GeoLite DB cannot be retrieved
     """
 
-    def distance(pfn):
+    def distance(pfn: str) -> float:
         url = urlparse(pfn)
         if url.scheme == 'root':
             # handle root proxy urls: root://10.0.0.1//root://192.168.1.1:1094//dpm/....
             sub_url = urlparse(url.path.lstrip('/'))
             if sub_url.scheme and sub_url.hostname:
                 url = sub_url
-        return __get_distance(url.hostname, client_location, ignore_error)
+        return __get_distance(url.hostname, client_location, ignore_error)  # type: ignore (hostname might be None)
 
     return list(sorted(dictreplica, key=distance))
 
 
-def sort_custom(dictreplica: dict, client_location: dict) -> list:
+def sort_custom(
+        dictreplica: dict[str, Any],
+        client_location: 'IPDict'
+) -> list[str]:
     """
     Return a list of replicas sorted according to the custom distance table.
     :param dictreplica: A dict with replicas as keys (URIs).
@@ -334,33 +360,3 @@ def sort_custom(dictreplica: dict, client_location: dict) -> list:
         return __get_distance_custom(dictreplica[pfn], client_location)
 
     return list(sorted(dictreplica, key=distance))
-
-
-def sort_closeness(dictreplica: dict, client_location: dict) -> list:
-    """
-    Return a list of replicas sorted by AGIS closeness. NOT IMPLEMENTED
-    :param dictreplica: A dict with replicas as keys (URIs).
-    :param client_location: Location dictionary containing {'ip', 'fqdn', 'site', 'latitude', 'longitude'}
-    """
-
-    return list(dictreplica.keys())
-
-
-def sort_ranking(dictreplica: dict, client_location: dict) -> list:
-    """
-    Return a list of replicas sorted by ranking metric. NOT IMPLEMENTED
-    :param dictreplica: A dict with replicas as keys (URIs).
-    :param client_location: Location dictionary containing {'ip', 'fqdn', 'site', 'latitude', 'longitude'}
-    """
-
-    return list(dictreplica.keys())
-
-
-def sort_dynamic(dictreplica: dict, client_location: dict) -> list:
-    """
-    Return a list of replicas sorted by dynamic network metrics. NOT IMPLEMENTED
-    :param dictreplica: A dict with replicas as keys (URIs).
-    :param client_location: Location dictionary containing {'ip', 'fqdn', 'site', 'latitude', 'longitude'}
-    """
-
-    return list(dictreplica.keys())

rucio/core/request.py

@@ -20,10 +20,8 @@ import math
 import random
 import threading
 import traceback
-import uuid
 from abc import ABCMeta, abstractmethod
 from collections import defaultdict, namedtuple
-from collections.abc import Iterable, Iterator, Mapping, Sequence
 from dataclasses import dataclass
 from typing import TYPE_CHECKING, Any, Optional, Union
 
@@ -31,7 +29,6 @@ from sqlalchemy import and_, delete, exists, insert, or_, select, update
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm import aliased
 from sqlalchemy.sql.expression import asc, false, func, null, true
-from sqlalchemy.sql.functions import coalesce
 
 from rucio.common.config import config_get_bool, config_get_int
 from rucio.common.constants import RseAttr
@@ -51,6 +48,8 @@ from rucio.db.sqla.util import temp_table_mngr
 RequestAndState = namedtuple('RequestAndState', ['request_id', 'request_state'])
 
 if TYPE_CHECKING:
+    import uuid
+    from collections.abc import Iterable, Iterator, Mapping, Sequence
 
     from sqlalchemy.engine import Row
     from sqlalchemy.orm import Session
@@ -298,7 +297,7 @@ def requeue_and_archive(
 @METRICS.count_it
 @transactional_session
 def queue_requests(
-    requests: Iterable[RequestDict],
+    requests: 'Iterable[RequestDict]',
     *,
     session: "Session",
     logger: LoggerFunction = logging.log
@@ -457,7 +456,7 @@ def list_and_mark_transfer_requests_and_source_replicas(
         limit: Optional[int] = None,
         activity: Optional[str] = None,
         older_than: Optional[datetime.datetime] = None,
-        rses: Optional[Sequence[str]] = None,
+        rses: Optional['Sequence[str]'] = None,
         request_type: Optional[list[RequestType]] = None,
         request_state: Optional[RequestState] = None,
         required_source_rse_attrs: Optional[list[str]] = None,
@@ -494,8 +493,6 @@ def list_and_mark_transfer_requests_and_source_replicas(
     if request_type is None:
         request_type = [RequestType.TRANSFER]
 
-    now = datetime.datetime.utcnow()
-
     sub_requests = select(
         models.Request.id,
         models.Request.request_type,
@@ -527,7 +524,11 @@ def list_and_mark_transfer_requests_and_source_replicas(
         models.ReplicationRule,
         models.Request.rule_id == models.ReplicationRule.id
     ).where(
-        coalesce(models.ReplicationRule.expires_at, now) >= now
+        or_(models.ReplicationRule.child_rule_id != null(),
+            and_(models.ReplicationRule.child_rule_id == null(),
+                 models.ReplicationRule.expires_at == null()),
+            and_(models.ReplicationRule.child_rule_id == null(),
+                 models.ReplicationRule.expires_at > datetime.datetime.utcnow()))
     ).join(
         models.RSE,
         models.RSE.id == models.Request.dest_rse_id
@@ -1035,7 +1036,7 @@ def transition_request_state(
 @METRICS.count_it
 @transactional_session
 def transition_requests_state_if_possible(
-    request_ids: Iterable[str],
+    request_ids: 'Iterable[str]',
     new_state: str,
     *,
     session: "Session",
@@ -1942,7 +1943,7 @@ class TransferStatsManager:
             resolution: datetime.timedelta,
             start_time: "Optional[datetime.datetime]" = None,
             end_time: "Optional[datetime.datetime]" = None
-    ) -> Iterator[tuple[datetime.datetime, datetime.datetime]]:
+    ) -> 'Iterator[tuple[datetime.datetime, datetime.datetime]]':
         """
         Iterates, back in time, over time intervals of length `resolution` which are fully
         included within the input interval (start_time, end_time).
@@ -2065,17 +2066,18 @@ def get_request_stats(
         activity: Optional[str] = None,
         *,
         session: "Session"
-) -> Sequence[
-    """Row[tuple[
-              Optional[InternalAccount],
-              RequestState,
-              uuid.UUID,
-              Optional[uuid.UUID],
-              Optional[str],
-              int,
-              Optional[int]
-    ]]"""
-]:
+) -> """Sequence[
+            Row[tuple[
+                Optional[InternalAccount],
+                RequestState,
+                uuid.UUID,
+                Optional[uuid.UUID],
+                Optional[str],
+                int,
+                Optional[int]
+                ]
+            ]
+]""":
     """
     Retrieve statistics about requests by destination, activity and state.
     """
@@ -2521,7 +2523,7 @@ def release_all_waiting_requests(
 def list_transfer_limits(
         *,
         session: "Session",
-) -> Iterator[dict[str, Any]]:
+) -> 'Iterator[dict[str, Any]]':
     stmt = select(
         models.TransferLimit
     )
@@ -2531,7 +2533,7 @@ def list_transfer_limits(
 
 
 def _sync_rse_transfer_limit(
-        limit_id: Union[str, uuid.UUID],
+        limit_id: Union[str, 'uuid.UUID'],
         desired_rse_ids: set[str],
         *,
         session: "Session",
@@ -2609,7 +2611,7 @@ def set_transfer_limit(
         waitings: Optional[int] = None,
         *,
         session: "Session",
-) -> uuid.UUID:
+) -> 'uuid.UUID':
     """
     Create or update a transfer limit
 
@@ -2991,12 +2993,12 @@ def get_source_rse(
 
 @stream_session
 def list_requests(
-    src_rse_ids: Sequence[str],
-    dst_rse_ids: Sequence[str],
-    states: Optional[Sequence[RequestState]] = None,
+    src_rse_ids: 'Sequence[str]',
+    dst_rse_ids: 'Sequence[str]',
+    states: Optional['Sequence[RequestState]'] = None,
     *,
     session: "Session"
-) -> Iterator[models.Request]:
+) -> 'Iterator[models.Request]':
     """
     List all requests in a specific state from a source RSE to a destination RSE.
 
@@ -3021,14 +3023,14 @@ def list_requests(
 
 @stream_session
 def list_requests_history(
-    src_rse_ids: Sequence[str],
-    dst_rse_ids: Sequence[str],
-    states: Optional[Sequence[RequestState]] = None,
+    src_rse_ids: 'Sequence[str]',
+    dst_rse_ids: 'Sequence[str]',
+    states: Optional['Sequence[RequestState]'] = None,
     offset: Optional[int] = None,
     limit: Optional[int] = None,
     *,
     session: "Session"
-) -> Iterator[models.RequestHistory]:
+) -> 'Iterator[models.RequestHistory]':
     """
     List all historical requests in a specific state from a source RSE to a destination RSE.