rucio 32.8.6__py3-none-any.whl → 35.8.0__py3-none-any.whl

rucio/daemons/c3po/algorithms/t2_free_space_only_pop.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,18 +14,22 @@
 
 import logging
 from operator import itemgetter
+from typing import TYPE_CHECKING, Any
 
 from rucio.common.config import config_get, config_get_int
 from rucio.common.exception import DataIdentifierNotFound
 from rucio.core.did import get_did
 from rucio.core.replica import list_dataset_replicas
-from rucio.core.rse import list_rse_attributes, get_rse_name
+from rucio.core.rse import get_rse_name, list_rse_attributes
 from rucio.core.rse_expression_parser import parse_expression
 from rucio.daemons.c3po.collectors.free_space import FreeSpaceCollector
 from rucio.daemons.c3po.utils.expiring_dataset_cache import ExpiringDatasetCache
 from rucio.daemons.c3po.utils.popularity import get_popularity
 from rucio.db.sqla.constants import ReplicaState
 
+if TYPE_CHECKING:
+    from rucio.common.types import InternalScope
+
 
 class PlacementAlgorithm:
     """
@@ -45,24 +48,24 @@ class PlacementAlgorithm:
 
         self.__setup_penalties()
 
-    def __setup_penalties(self):
+    def __setup_penalties(self) -> None:
         self._penalties = {}
         for rse_id in self._rses:
             self._penalties[rse_id] = 1.0
 
-    def __update_penalties(self):
+    def __update_penalties(self) -> None:
         for rse_id, penalty in self._penalties.items():
             if penalty > 1.0:
                 self._penalties[rse_id] = penalty - 1
 
-    def place(self, did):
+    def place(self, did: tuple['InternalScope', str]) -> dict[str, Any]:
         self.__update_penalties()
-        decision = {'did': '{}:{}'.format(did[0].internal, did[1])}
+        decision: dict[str, Any] = {'did': '{}:{}'.format(did[0].internal, did[1])}
         if (self._added_cache.check_dataset(decision['did'])):
             decision['error_reason'] = 'already added replica for this did in the last 24h'
             return decision
 
-        if (not did[0].external.startswith('data')) and (not did[0].external.startswith('mc')):
+        if (did[0].external is not None) and (not did[0].external.startswith('data')) and (not did[0].external.startswith('mc')):
             decision['error_reason'] = 'not a data or mc dataset'
             return decision
 

rucio/daemons/c3po/algorithms/t2_free_space_only_pop_with_network.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,12 +14,14 @@
 
 import logging
 from operator import itemgetter
+from typing import TYPE_CHECKING, Any
 
 from rucio.common.config import config_get, config_get_int
+from rucio.common.constants import RseAttr
 from rucio.common.exception import DataIdentifierNotFound
 from rucio.core.did import get_did
 from rucio.core.replica import list_dataset_replicas
-from rucio.core.rse import list_rse_attributes, get_rse, get_rse_name
+from rucio.core.rse import get_rse, get_rse_name, list_rse_attributes
 from rucio.core.rse_expression_parser import parse_expression
 from rucio.daemons.c3po.collectors.free_space import FreeSpaceCollector
 from rucio.daemons.c3po.collectors.network_metrics import NetworkMetricsCollector
@@ -30,12 +31,26 @@ from rucio.daemons.c3po.utils.popularity import get_popularity
 from rucio.daemons.c3po.utils.timeseries import RedisTimeSeries
 from rucio.db.sqla.constants import ReplicaState
 
+if TYPE_CHECKING:
+    from rucio.common.types import InternalScope
+
 
 class PlacementAlgorithm:
     """
     Placement algorithm that focusses on free space on T2 DATADISK RSEs. It incorporates network metrics for placement decisions.
     """
-    def __init__(self, datatypes, dest_rse_expr, max_bytes_hour, max_files_hour, max_bytes_hour_rse, max_files_hour_rse, min_popularity, min_recent_requests, max_replicas):
+    def __init__(
+            self,
+            datatypes: str,
+            dest_rse_expr: str,
+            max_bytes_hour: int,
+            max_files_hour: int,
+            max_bytes_hour_rse: int,
+            max_files_hour_rse: int,
+            min_popularity: int,
+            min_recent_requests: int,
+            max_replicas: int
+    ):
         self._fsc = FreeSpaceCollector()
         self._nmc = NetworkMetricsCollector()
         self._added_cache = ExpiringDatasetCache(config_get('c3po', 'redis_host'), config_get_int('c3po', 'redis_port'), timeout=86400)
@@ -61,14 +76,14 @@ class PlacementAlgorithm:
             rse_attrs = list_rse_attributes(rse_id=rse['id'])
             rse_attrs['rse_id'] = rse['id']
             self._rses[rse['id']] = rse_attrs
-            self._sites[rse_attrs['site']] = rse_attrs
+            self._sites[rse_attrs[RseAttr.SITE]] = rse_attrs
 
         self._dst_penalties = {}
         self._src_penalties = {}
 
         self._print_params()
 
-    def _print_params(self):
+    def _print_params(self) -> None:
         logging.debug('Parameter Overview:')
         logging.debug('Algorithm: t2_free_space_only_pop_with_network')
         logging.debug('Datatypes: %s' % ','.join(self._datatypes))
@@ -77,7 +92,7 @@ class PlacementAlgorithm:
         logging.debug('Min recent requests / Min popularity: %d / %d' % (self._min_recent_requests, self._min_popularity))
         logging.debug('Max existing replicas: %d' % self._max_replicas)
 
-    def __update_penalties(self):
+    def __update_penalties(self) -> None:
         for rse_id, penalty in self._dst_penalties.items():
             if penalty < 100.0:
                 self._dst_penalties[rse_id] += 10.0
@@ -86,13 +101,13 @@ class PlacementAlgorithm:
             if penalty < 100.0:
                 self._src_penalties[rse_id] += 10.0
 
-    def check_did(self, did):
-        decision = {'did': '{}:{}'.format(did[0].internal, did[1])}
+    def check_did(self, did: tuple['InternalScope', str]) -> dict[str, Any]:
+        decision: dict[str, Any] = {'did': '{}:{}'.format(did[0].internal, did[1])}
         if (self._added_cache.check_dataset(decision['did'])):
             decision['error_reason'] = 'already added replica for this did in the last 24h'
             return decision
 
-        if (not did[0].external.startswith('data')) and (not did[0].external.startswith('mc')):
+        if (did[0].external is not None) and (not did[0].external.startswith('data')) and (not did[0].external.startswith('mc')):
             decision['error_reason'] = 'not a data or mc dataset'
             return decision
 
@@ -141,13 +156,13 @@ class PlacementAlgorithm:
             decision['error_reason'] = 'problems connecting to ES'
             return decision
 
-        if (last_accesses < self._min_recent_requests) and (pop < self._min_popularity):
+        if (last_accesses < self._min_recent_requests) and (decision['popularity'] < self._min_popularity):
             decision['error_reason'] = 'did not popular enough'
             return decision
 
         return decision
 
-    def place(self, did):
+    def place(self, did: tuple['InternalScope', str]) -> dict[str, Any]:
         self.__update_penalties()
         self._added_bytes.trim()
         self._added_files.trim()
@@ -166,10 +181,10 @@ class PlacementAlgorithm:
         for rep in reps:
             rse_attr = list_rse_attributes(rep['rse_id'])
             src_rse_id = rep['rse_id']
-            if 'site' not in rse_attr:
+            if RseAttr.SITE not in rse_attr:
                 continue
 
-            src_site = rse_attr['site']
+            src_site = rse_attr[RseAttr.SITE]
             src_rse_info = get_rse(rse_id=src_rse_id)
 
             if 'type' not in rse_attr:
@@ -189,10 +204,10 @@ class PlacementAlgorithm:
                     net_metrics = self._nmc.getMbps(src_site, metric_type)
                     if net_metrics:
                         break
-                if len(net_metrics) == 0:
+                if len(net_metrics) == 0:  # type: ignore
                     continue
                 available_reps[src_rse_id] = {}
-                for dst_site, mbps in net_metrics.items():
+                for dst_site, mbps in net_metrics.items():  # type: ignore
                     if src_site == dst_site:
                         continue
                     if dst_site in self._sites:

rucio/daemons/c3po/c3po.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -24,7 +23,7 @@ from json import dumps
 from queue import Queue
 from threading import Event, Thread
 from time import sleep
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Optional
 from uuid import uuid4
 
 from requests import post
@@ -43,13 +42,17 @@ from rucio.daemons.c3po.collectors.workload import WorkloadCollector
 
 if TYPE_CHECKING:
     from types import FrameType
-    from typing import Optional
 
 GRACEFUL_STOP = Event()
 DAEMON_NAME = 'c3po'
 
 
-def read_free_space(once=False, thread=0, waiting_time=1800, sleep_time=10):
+def read_free_space(
+        once: bool = False,
+        thread: int = 0,
+        waiting_time: int = 1800,
+        sleep_time: int = 10
+) -> None:
     """
     Thread to collect the space usage information for RSEs.
     """
@@ -66,7 +69,12 @@ def read_free_space(once=False, thread=0, waiting_time=1800, sleep_time=10):
         timer = 0
 
 
-def read_workload(once=False, thread=0, waiting_time=1800, sleep_time=10):
+def read_workload(
+        once: bool = False,
+        thread: int = 0,
+        waiting_time: int = 1800,
+        sleep_time: int = 10
+) -> None:
     """
     Thread to collect the workload information from PanDA.
     """
@@ -83,7 +91,12 @@ def read_workload(once=False, thread=0, waiting_time=1800, sleep_time=10):
         timer = 0
 
 
-def print_workload(once=False, thread=0, waiting_time=600, sleep_time=10):
+def print_workload(
+        once: bool = False,
+        thread: int = 0,
+        waiting_time: int = 600,
+        sleep_time: int = 10
+) -> None:
     """
     Thread to regularly output the workload to logs for debugging.
     """
@@ -101,7 +114,13 @@ def print_workload(once=False, thread=0, waiting_time=600, sleep_time=10):
         timer = 0
 
 
-def read_dids(once=False, thread=0, did_collector=None, waiting_time=60, sleep_time=10):
+def read_dids(
+        once: bool = False,
+        thread: int = 0,
+        did_collector: Optional[JediDIDCollector] = None,
+        waiting_time: int = 60,
+        sleep_time: int = 10
+) -> None:
     """
     Thread to collect DIDs for the placement algorithm.
     """
@@ -112,33 +131,41 @@ def read_dids(once=False, thread=0, did_collector=None, waiting_time=60, sleep_t
             sleep(sleep_time)
             continue
 
-        did_collector.get_dids()
+        if did_collector is not None:
+            did_collector.get_dids()
         timer = 0
 
 
-def add_rule(client, did, src_rse, dst_rse):
+def add_rule(
+        client: Client,
+        did: dict[str, str],
+        src_rse: str,
+        dst_rse: str
+) -> None:
     logging.debug('add rule for %s from %s to %s' % (did, src_rse, dst_rse))
     r = client.add_replication_rule([did, ], 1, dst_rse, lifetime=604800, account='c3po', source_replica_expression=src_rse, activity='Data Brokering', asynchronous=True)
     logging.debug(r)
 
 
-def place_replica(once=False,
-                  thread=0,
-                  did_queue=None,
-                  waiting_time=100,
-                  dry_run=False,
-                  sampling=False,
-                  algorithms='t2_free_space_only_pop_with_network',
-                  datatypes='NTUP,DAOD',
-                  dest_rse_expr='type=DATADISK',
-                  max_bytes_hour=100000000000000,
-                  max_files_hour=100000,
-                  max_bytes_hour_rse=50000000000000,
-                  max_files_hour_rse=10000,
-                  min_popularity=8,
-                  min_recent_requests=5,
-                  max_replicas=5,
-                  sleep_time=10):
+def place_replica(
+        did_queue: Queue,
+        once: bool = False,
+        thread: int = 0,
+        waiting_time: int = 100,
+        dry_run: bool = False,
+        sampling: bool = False,
+        algorithms: str = 't2_free_space_only_pop_with_network',
+        datatypes: str = 'NTUP,DAOD',
+        dest_rse_expr: str = 'type=DATADISK',
+        max_bytes_hour: int = 100000000000000,
+        max_files_hour: int = 100000,
+        max_bytes_hour_rse: int = 50000000000000,
+        max_files_hour_rse: int = 10000,
+        min_popularity: int = 8,
+        min_recent_requests: int = 5,
+        max_replicas: int = 5,
+        sleep_time: int = 10
+) -> None:
     """
     Thread to run the placement algorithm to decide if and where to put new replicas.
     """
@@ -149,17 +176,17 @@ def place_replica(once=False,
         if 'algorithms' in c3po_options:
             algorithms = config_get('c3po', 'algorithms')
 
-        algorithms = algorithms.split(',')
+        algorithms_list = algorithms.split(',')
 
         if not dry_run:
-            if len(algorithms) != 1:
+            if len(algorithms_list) != 1:
                 logging.error('Multiple algorithms are only allowed in dry_run mode')
                 return
             client = Client(auth_type='x509_proxy', account='c3po', creds={'client_proxy': '/opt/rucio/etc/ddmadmin.long.proxy'})
 
         vo = client.vo
         instances = {}
-        for algorithm in algorithms:
+        for algorithm in algorithms_list:
             module_path = 'rucio.daemons.c3po.algorithms.' + algorithm
             module = __import__(module_path, globals(), locals(), ['PlacementAlgorithm'])
             instance = module.PlacementAlgorithm(datatypes, dest_rse_expr, max_bytes_hour, max_files_hour, max_bytes_hour_rse, max_files_hour_rse, min_popularity, min_recent_requests, max_replicas)
@@ -245,7 +272,7 @@ def place_replica(once=False,
                     if (not dry_run) and create_rule:
                         # DO IT!
                         try:
-                            add_rule(client, {'scope': did[0].external, 'name': did[1]}, decision.get('source_rse'), decision.get('destination_rse'))
+                            add_rule(client, {'scope': did[0].external, 'name': did[1]}, decision.get('source_rse'), decision.get('destination_rse'))  # type: ignore
                         except exception.RucioException as e:
                             logging.debug(e)
 
@@ -254,34 +281,36 @@ def place_replica(once=False,
         logging.critical(e)
 
 
-def stop(signum: "Optional[int]" = None, frame: "Optional[FrameType]" = None) -> None:
+def stop(signum: Optional[int] = None, frame: Optional['FrameType'] = None) -> None:
     """
     Graceful exit.
     """
     GRACEFUL_STOP.set()
 
 
-def run(once=False,
-        threads=1,
-        only_workload=False,
-        dry_run=False,
-        sampling=False,
-        algorithms='t2_free_space_only_pop_with_network',
-        datatypes='NTUP,DAOD',
-        dest_rse_expr='type=DATADISK',
-        max_bytes_hour=100000000000000,
-        max_files_hour=100000,
-        max_bytes_hour_rse=50000000000000,
-        max_files_hour_rse=10000,
-        min_popularity=8,
-        min_recent_requests=5,
-        max_replicas=5,
-        waiting_time_read_free_space=1800,
-        waiting_time_read_workload=1800,
-        waiting_time_print_workload=600,
-        waiting_time_read_dids=60,
-        waiting_time_place_replica=100,
-        sleep_time=10):
+def run(
+        once: bool = False,
+        threads: int = 1,
+        only_workload: bool = False,
+        dry_run: bool = False,
+        sampling: bool = False,
+        algorithms: str = 't2_free_space_only_pop_with_network',
+        datatypes: str = 'NTUP,DAOD',
+        dest_rse_expr: str = 'type=DATADISK',
+        max_bytes_hour: int = 100000000000000,
+        max_files_hour: int = 100000,
+        max_bytes_hour_rse: int = 50000000000000,
+        max_files_hour_rse: int = 10000,
+        min_popularity: int = 8,
+        min_recent_requests: int = 5,
+        max_replicas: int = 5,
+        waiting_time_read_free_space: int = 1800,
+        waiting_time_read_workload: int = 1800,
+        waiting_time_print_workload: int = 600,
+        waiting_time_read_dids: int = 60,
+        waiting_time_place_replica: int = 100,
+        sleep_time: int = 10
+) -> None:
     """
     Starts up the main thread
     """

rucio/daemons/c3po/collectors/__init__.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

rucio/daemons/c3po/collectors/agis.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,17 +13,18 @@
 # limitations under the License.
 
 from json import loads
+from typing import Optional
 
 from requests import get
 
 from rucio.common.config import config_get
 
 
-class MappingCollector(object):
+class MappingCollector:
     """
     Provides mappings from PanDA / DDM resources to ATLAS sites and back.
     """
-    class _MappingCollector(object):
+    class _MappingCollector:
         '''
         _MappingCollector
         '''
@@ -36,7 +36,7 @@ class MappingCollector(object):
             self._fetch_panda_mapping()
             self._fetch_ddm_mapping()
 
-        def _fetch_panda_mapping(self):
+        def _fetch_panda_mapping(self) -> None:
             '''
             _fetch_panda_mapping
             '''
@@ -51,7 +51,7 @@ class MappingCollector(object):
                     self.site_to_panda[entry['atlas_site']] = []
                 self.site_to_panda[entry['atlas_site']].append(entry['panda_resource'])
 
-        def _fetch_ddm_mapping(self):
+        def _fetch_ddm_mapping(self) -> None:
             '''
             _fetch_ddm_mapping
             '''
@@ -75,34 +75,34 @@ class MappingCollector(object):
         if not MappingCollector.instance:
             MappingCollector.instance = MappingCollector._MappingCollector()
 
-    def ddm_to_site(self, ddm):
+    def ddm_to_site(self, ddm: str) -> Optional[str]:
         '''
         ddm_to_site
         '''
-        if ddm not in self.instance.ddm_to_site:
+        if ddm not in self.instance.ddm_to_site:  # type: ignore
             return None
-        return self.instance.ddm_to_site[ddm]
+        return self.instance.ddm_to_site[ddm]  # type: ignore
 
-    def panda_to_site(self, panda):
+    def panda_to_site(self, panda: str) -> Optional[str]:
         '''
         panda_to_site
         '''
-        if panda not in self.instance.panda_to_site:
+        if panda not in self.instance.panda_to_site:  # type: ignore
             return None
-        return self.instance.panda_to_site[panda]
+        return self.instance.panda_to_site[panda]  # type: ignore
 
-    def site_to_ddm(self, site):
+    def site_to_ddm(self, site: str) -> Optional[str]:
         '''
         site_to_ddm
         '''
-        if site not in self.instance.site_to_ddm:
+        if site not in self.instance.site_to_ddm:  # type: ignore
             return None
-        return self.instance.site_to_ddm[site]
+        return self.instance.site_to_ddm[site]  # type: ignore
 
-    def site_to_panda(self, site):
+    def site_to_panda(self, site: str) -> Optional[str]:
         '''
         site_to_panda
         '''
-        if site not in self.instance.site_to_panda:
+        if site not in self.instance.site_to_panda:  # type: ignore
             return None
-        return self.instance.site_to_panda[site]
+        return self.instance.site_to_panda[site]  # type: ignore

rucio/daemons/c3po/collectors/free_space.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -17,15 +16,22 @@
 Collector to get the SRM free and used information for DATADISK RSEs.
 """
 
-from rucio.db.sqla.models import RSEUsage, RSEAttrAssociation
+from typing import TYPE_CHECKING, Optional
+
+from sqlalchemy import and_, select
+
+from rucio.db.sqla.models import RSEAttrAssociation, RSEUsage
 from rucio.db.sqla.session import read_session
 
+if TYPE_CHECKING:
+    from sqlalchemy.orm import Session
+
 
-class FreeSpaceCollector(object):
+class FreeSpaceCollector:
     """
     Collector to get the SRM free and used information for DATADISK RSEs.
     """
-    class _FreeSpaceCollector(object):
+    class _FreeSpaceCollector:
         """
         Hidden implementation
         """
@@ -33,14 +39,27 @@ class FreeSpaceCollector(object):
             self.rses = {}
 
         @read_session
-        def _collect_free_space(self, *, session=None):
+        def _collect_free_space(
+            self,
+            *,
+            session: Optional["Session"] = None
+        ) -> None:
             """
             Retrieve free space from database
             """
-            query = session.query(RSEUsage.rse_id, RSEUsage.free, RSEUsage.used).\
-                join(RSEAttrAssociation, RSEUsage.rse_id == RSEAttrAssociation.rse_id).\
-                filter(RSEUsage.source == 'storage').filter(RSEAttrAssociation.key == 'type', RSEAttrAssociation.value == 'DATADISK')
-            for rse_id, free, used in query:
+            stmt = select(
+                RSEUsage.rse_id,
+                RSEUsage.free,
+                RSEUsage.used
+            ).join(
+                RSEAttrAssociation,
+                RSEAttrAssociation.rse_id == RSEUsage.rse_id
+            ).where(
+                and_(RSEUsage.source == 'storage',
+                     RSEAttrAssociation.key == 'type',
+                     RSEAttrAssociation.value == 'DATADISK'),
+            )
+            for rse_id, free, used in session.execute(stmt).all():  # type: ignore (session could be None)
                 self.rses[rse_id] = {'total': used + free, 'used': used, 'free': free}
 
     instance = None
@@ -49,14 +68,14 @@ class FreeSpaceCollector(object):
         if not FreeSpaceCollector.instance:
             FreeSpaceCollector.instance = FreeSpaceCollector._FreeSpaceCollector()
 
-    def collect_free_space(self):
+    def collect_free_space(self) -> None:
         """
         Execute the free space collector
         """
-        self.instance._collect_free_space()
+        self.instance._collect_free_space()  # type: ignore
 
-    def get_rse_space(self):
+    def get_rse_space(self) -> dict[str, dict[str, int]]:
         """
         Return the RSE space
         """
-        return self.instance.rses
+        return self.instance.rses  # type: ignore

rucio/daemons/c3po/collectors/jedi_did.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,17 +13,27 @@
 # limitations under the License.
 
 import logging
+from typing import TYPE_CHECKING, Optional
 
 from rucio.db.sqla.session import read_session
 
+if TYPE_CHECKING:
+    from queue import Queue
 
-class JediDIDCollector():
-    def __init__(self, queue):
+    from sqlalchemy.orm import Session
+
+
+class JediDIDCollector:
+    def __init__(self, queue: "Queue"):
         self.queue = queue
         self.max_tid = 0
 
     @read_session
-    def get_dids(self, *, session=None):
+    def get_dids(
+        self,
+        *,
+        session: Optional["Session"] = None
+    ) -> None:
         query = """select t.jeditaskid, t.username, t.status, d.datasetname from ATLAS_PANDA.JEDI_TASKS t
         inner join ATLAS_PANDA.JEDI_DATASETS d
         on t.jeditaskid = d.jeditaskid
@@ -32,7 +41,7 @@ class JediDIDCollector():
         and d.type = 'input'
         order by d.jeditaskid asc"""
 
-        tasks = session.execute(query)
+        tasks = session.execute(query)  # type: ignore
 
         for t in tasks.fetchall():
             status = t[2]