rucio 32.8.6__py3-none-any.whl → 35.8.0__py3-none-any.whl

rucio/daemons/c3po/collectors/mock_did.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -17,19 +16,25 @@
 Mock DID collector
 """
 
-from random import choice
+from secrets import choice
+from typing import TYPE_CHECKING
 
+if TYPE_CHECKING:
+    from multiprocessing import Queue
 
-class MockDIDCollector(object):
+    from _typeshed import FileDescriptorOrPath
+
+
+class MockDIDCollector:
     """
     Simple collector that reads dids from a file. Used to
     test the interface.
     """
-    def __init__(self, queue):
+    def __init__(self, queue: "Queue"):
         self._queue = queue
         self._read_file('/opt/rucio/etc/dids_mc15_13TeV.csv')
 
-    def _read_file(self, infile):
+    def _read_file(self, infile: "FileDescriptorOrPath") -> None:
         dids = []
         with open(infile, 'r') as f:
             f.readline()
@@ -41,6 +46,6 @@ class MockDIDCollector(object):
 
         self._dids = tuple(dids)
 
-    def get_dids(self):
+    def get_dids(self) -> None:
         did = choice(self._dids)
         self._queue.put(did)

rucio/daemons/c3po/collectors/network_metrics.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,13 +13,14 @@
 # limitations under the License.
 
 from json import loads
+from typing import Optional
 
 from redis import StrictRedis
 
 from rucio.common.config import config_get, config_get_int
 
 
-class NetworkMetricsCollector(object):
+class NetworkMetricsCollector:
     """
     Collector to get the bandwidth metrics between two sites.
     """
@@ -29,7 +29,11 @@ class NetworkMetricsCollector(object):
         self._r = StrictRedis(host=config_get('c3po-network-metrics', 'redis_host'), port=config_get_int('c3po-network-metrics', 'redis_port'))
         self._prefix = config_get('c3po-network-metrics', 'prefix')
 
-    def getMbps(self, src, type_):
+    def getMbps(
+            self,
+            src: str,
+            type_: str
+    ) -> Optional[dict[str, float]]:
         pattern = "%s#%s:*" % (self._prefix, src)
         keys = self._r.keys(pattern=pattern)
         if len(keys) == 0:
@@ -52,7 +56,11 @@ class NetworkMetricsCollector(object):
 
         return ret
 
-    def getQueuedFiles(self, src, dst):
+    def getQueuedFiles(
+            self,
+            src: str,
+            dst: str
+    ) -> int:
         key = "%s#%s:%s" % (self._prefix, src, dst)
         activities = loads(self._r.get(key)).get('files', {}).get('queued', {}).get('total', {})
 

rucio/daemons/c3po/collectors/workload.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -27,17 +26,20 @@ from rucio.common.config import config_get, config_get_int
 from rucio.daemons.c3po.utils.timeseries import RedisTimeSeries
 
 
-class WorkloadCollector(object):
+class WorkloadCollector:
     """
     Collector to retrieve the workload from PanDA. It stores it as a time series in Redis and provides
     the average and maximum number of running jobs for a sliding window.
     """
 
-    class __WorkloadCollector(object):
+    class __WorkloadCollector:
         """
         Private class needed implement singleton.
         """
-        def __init__(self, delete_keys=False):
+        def __init__(
+                self,
+                delete_keys: bool = False
+        ):
             self._avg_jobs = {}
             self._cur_jobs = {}
             self._max_jobs = {}
@@ -49,7 +51,7 @@ class WorkloadCollector(object):
                 self._tms.delete_keys()
             self.reload_cache()
 
-        def reload_cache(self):
+        def reload_cache(self) -> None:
             self._tms.trim()
 
             for key in self._tms.get_keys():
@@ -61,7 +63,7 @@ class WorkloadCollector(object):
                     self._max_jobs[site] = max(job_series)
                     self._cur_jobs[site] = job_series[-1]
 
-        def collect_workload(self):
+        def collect_workload(self) -> None:
             start = time()
             resp = get(self._request_url, headers=self._request_headers)
             logging.debug("PanDA response took %fs" % (time() - start))
@@ -88,23 +90,23 @@ class WorkloadCollector(object):
         if not WorkloadCollector.instance:
             WorkloadCollector.instance = WorkloadCollector.__WorkloadCollector()
 
-    def get_avg_jobs(self, site):
-        return self.instance._avg_jobs[site]
+    def get_avg_jobs(self, site: str) -> float:
+        return self.instance._avg_jobs[site]  # type: ignore
 
-    def get_max_jobs(self, site):
-        return self.instance._max_jobs[site]
+    def get_max_jobs(self, site: str) -> int:
+        return self.instance._max_jobs[site]  # type: ignore
 
-    def get_cur_jobs(self, site):
-        return self.instance._cur_jobs[site]
+    def get_cur_jobs(self, site: str) -> int:
+        return self.instance._cur_jobs[site]  # type: ignore
 
-    def get_sites(self):
-        return list(self.instance._avg_jobs.keys())
+    def get_sites(self) -> list[str]:
+        return list(self.instance._avg_jobs.keys())  # type: ignore
 
-    def get_job_info(self, site):
+    def get_job_info(self, site: str) -> tuple[int, float, int]:
         return (self.get_cur_jobs(site), self.get_avg_jobs(site), self.get_max_jobs(site))
 
-    def get_series(self, site):
-        return self.instance._tms.get_series(site)
+    def get_series(self, site: str) -> tuple[int]:
+        return self.instance._tms.get_series(site)  # type: ignore
 
-    def collect_workload(self):
-        self.instance.collect_workload()
+    def collect_workload(self) -> None:
+        self.instance.collect_workload()  # type: ignore

rucio/daemons/c3po/utils/__init__.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

rucio/daemons/c3po/utils/dataset_cache.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,26 +12,37 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from typing import TYPE_CHECKING
 from uuid import uuid4
 
 from rucio.daemons.c3po.utils.timeseries import RedisTimeSeries
 
+if TYPE_CHECKING:
+    from rucio.common.types import InternalScope
 
-class DatasetCache(object):
+
+class DatasetCache:
     """
     Utility to count the accesses of the datasets during the last day.
     """
-    def __init__(self, redis_host, redis_port, timeout=1, prefix='did_cache', delete_keys=False):
+    def __init__(
+            self,
+            redis_host: str,
+            redis_port: int,
+            timeout: int = 1,
+            prefix: str = 'did_cache',
+            delete_keys: bool = False
+    ):
         self._prefix = prefix + '_' + str(uuid4()).split('-')[0]
         self._tms = RedisTimeSeries(redis_host, redis_port, timeout, self._prefix)
 
         if delete_keys:
             self._tms.delete_keys()
 
-    def add_did(self, did):
+    def add_did(self, did: tuple['InternalScope', str]) -> None:
         self._tms.add_point('{}_{}'.format(did[0].internal, did[1]), 1)
 
-    def get_did(self, did):
+    def get_did(self, did: tuple['InternalScope', str]) -> int:
         self._tms.trim()
 
         series = self._tms.get_series('{}_{}'.format(did[0].internal, did[1]))

rucio/daemons/c3po/utils/expiring_dataset_cache.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -21,22 +20,34 @@ from uuid import uuid4
 from redis import StrictRedis
 
 
-class ExpiringDatasetCache(object):
+class ExpiringDatasetCache:
     """
     Cache with expiring values to keep track of recently created replicas.
     """
-    def __init__(self, redis_host, redis_port, timeout=1, prefix='expiring_did_cache'):
+    def __init__(
+            self,
+            redis_host: str,
+            redis_port: int,
+            timeout: int = 1,
+            prefix: str = 'expiring_did_cache'
+    ):
         self._redis = StrictRedis(host=redis_host, port=redis_port)
         self._prefix = prefix + '_' + str(uuid4()).split('-')[0]
         self._timeout = timeout
 
-    def add_dataset(self, dataset):
+    def add_dataset(
+            self,
+            dataset: str
+    ) -> None:
         """ Adds a datasets to cache with lifetime """
         key = ':'.join((self._prefix, dataset))
         self._redis.set(key, 1)
         self._redis.expire(key, self._timeout)
 
-    def check_dataset(self, dataset):
+    def check_dataset(
+            self,
+            dataset: str
+    ) -> bool:
         """ Checks if dataset is still in cache """
         key = ':'.join((self._prefix, dataset))
         if self._redis.get(key) is None:

rucio/daemons/c3po/utils/expiring_list.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -21,12 +20,12 @@ from collections import deque
 from threading import Lock, Timer
 
 
-class ExpiringList(object):
+class ExpiringList:
     """
     Simple list with time based element expiration
     """
 
-    def __init__(self, timeout=1):
+    def __init__(self, timeout: int = 1):
         self._lock = Lock()
         self._timeout = timeout
         self._items = deque()
@@ -38,26 +37,26 @@ class ExpiringList(object):
             self._items.append(item)
             Timer(self._timeout, self._expire).start()
 
-    def __len__(self):
+    def __len__(self) -> int:
         """
         Return number of active events
         """
         with self._lock:
             return len(self._items)
 
-    def _expire(self):
+    def _expire(self) -> None:
         """
         Remove any expired events
         """
         with self._lock:
             self._items.popleft()
 
-    def to_set(self):
+    def to_set(self) -> set:
         """
         Return items as a set
         """
         return set(self._items)
 
-    def __str__(self):
+    def __str__(self) -> str:
         with self._lock:
             return str(self._items)

rucio/daemons/c3po/utils/popularity.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,12 +14,16 @@
 
 import logging
 from json import dumps, loads
+from typing import TYPE_CHECKING, Optional
 
 from requests import post
 from requests.auth import HTTPBasicAuth
 
 from rucio.common.config import config_get, config_get_options
 
+if TYPE_CHECKING:
+    from rucio.common.types import InternalScope
+
 ELASTIC_URL = config_get('es-atlas', 'url')
 
 ELASTIC_OPTIONS = config_get_options('es-atlas')
@@ -37,7 +40,7 @@ else:
 URL = ELASTIC_URL + '/atlas_rucio-popularity-*/_search'
 
 
-def get_popularity(did):
+def get_popularity(did: tuple['InternalScope', str]) -> Optional[int]:
     """
     Query the popularity for a given DID in the ElasticSearch popularity db.
     """

rucio/daemons/c3po/utils/timeseries.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -18,59 +17,73 @@ Redis time series abstraction
 """
 
 from time import time
+from typing import TYPE_CHECKING
 
 from redis import StrictRedis
 
+if TYPE_CHECKING:
+    from redis.commands.core import ResponseT
 
-class RedisTimeSeries(object):
+
+class RedisTimeSeries:
     """
     Redis time series abstraction
     """
 
-    def __init__(self, redis_host, redis_port, window, prefix):
+    def __init__(
+            self,
+            redis_host: str,
+            redis_port: int,
+            window: int,
+            prefix: str
+    ):
         self._redis = StrictRedis(host=redis_host, port=redis_port)
         self._prefix = prefix
         self._window = window * 1000000
 
-    def add_point(self, key, value):
+    def add_point(
+            self,
+            key: str,
+            value: int
+    ) -> None:
         """
         Add a point
         """
         r_key = self._prefix + key
         score = int(time() * 1000000)
-        self._redis.zadd(r_key, score, "%d:%d" % (value, score))
+        self._redis.zadd(r_key, score, "%d:%d" % (value, score))  # type: ignore
 
-    def get_series(self, key):
+    def get_series(self, key: str) -> tuple[int]:
         """
         Return a time series tuple
         """
         r_key = self._prefix + key
         r_series = self._redis.zrange(r_key, 0, -1)
         series = []
-        for val in r_series:
+        for val in r_series:  # type: ignore
             values, _ = val.split(':')
             series.append(int(values))
 
         return tuple(series)
 
-    def trim(self):
+    def trim(self) -> None:
         """
         Trim the time series
         """
         now = time()
         max_score = int(now * 1000000 - self._window)
-        for key in self.get_keys():
+        for key in self.get_keys():  # type: ignore
             self._redis.zremrangebyscore(key, 0, max_score)
 
-    def get_keys(self):
+    def get_keys(self) -> "ResponseT":
         """
         Return matching keys
         """
         return self._redis.keys(pattern=self._prefix + "*")
 
-    def delete_keys(self):
+    def delete_keys(self) -> None:
         """
         Delete keys
         """
-        for key in self.get_keys():
+        for key in self.get_keys():  # type: ignore
             self._redis.zremrangebyrank(key, 0, -1)

rucio/daemons/cache/__init__.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

rucio/daemons/cache/consumer.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -22,21 +21,23 @@ import logging
 import threading
 import time
 from traceback import format_exc
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Optional
 
 import rucio.db.sqla.util
 from rucio.common import exception
-from rucio.common.config import config_get, config_get_int, config_get_bool, config_get_list
-from rucio.common.logging import setup_logging, formatted_logger
+from rucio.common.config import config_get, config_get_bool, config_get_int, config_get_list
+from rucio.common.logging import formatted_logger, setup_logging
 from rucio.common.stomp_utils import StompConnectionManager
-from rucio.common.types import InternalScope
+from rucio.common.types import InternalScope, LoggerFunction
 from rucio.core.monitor import MetricManager
 from rucio.core.rse import get_rse_id
 from rucio.core.volatile_replica import add_volatile_replicas, delete_volatile_replicas
 
 if TYPE_CHECKING:
     from types import FrameType
-    from typing import Optional
+
+    from stomp import Connection
+    from stomp.utils import Frame
 
 logging.getLogger("stomp").setLevel(logging.CRITICAL)
 
@@ -45,12 +46,17 @@ GRACEFUL_STOP = threading.Event()
 DAEMON_NAME = 'cache-consumer'
 
 
-class AMQConsumer(object):
+class AMQConsumer:
     """
     class Consumer
     """
 
-    def __init__(self, broker, conn, logger):
+    def __init__(
+            self,
+            broker: str,
+            conn: "Connection",
+            logger: "LoggerFunction"
+    ):
         """
         __init__
         """
@@ -59,23 +65,23 @@ class AMQConsumer(object):
         self.__logger = logger
 
     @METRICS.count_it
-    def on_heartbeat_timeout(self):
+    def on_heartbeat_timeout(self) -> None:
         self.__conn.disconnect()
 
     @METRICS.count_it
-    def on_error(self, frame):
+    def on_error(self, frame: "Frame") -> None:
         """
         on_error
         """
         self.__logger(logging.ERROR, 'Message receive error: [%s] %s' % (self.__broker, frame.body))
 
     @METRICS.count_it
-    def on_message(self, frame):
+    def on_message(self, frame: "Frame") -> None:
         """
         on_message
         """
         try:
-            msg = json.loads(frame.body)
+            msg = json.loads(frame.body)  # type: ignore
             self.__logger(logging.DEBUG, 'Message received: %s ' % msg)
             if isinstance(msg, dict) and 'operation' in msg.keys():
                 for f in msg['files']:
@@ -101,7 +107,7 @@ class AMQConsumer(object):
             self.__logger(logging.ERROR, str(format_exc()))
 
 
-def consumer(id_, num_thread=1):
+def consumer(id_: int, num_thread: int = 1) -> None:
     """
     Main loop to consume messages from the Rucio Cache producer.
     """
@@ -161,7 +167,7 @@ def consumer(id_, num_thread=1):
     logger(logging.INFO, 'graceful stop done')
 
 
-def stop(signum: "Optional[int]" = None, frame: "Optional[FrameType]" = None) -> None:
+def stop(signum: Optional[int] = None, frame: Optional["FrameType"] = None) -> None:
     """
     Graceful exit.
     """
@@ -169,7 +175,7 @@ def stop(signum: "Optional[int]" = None, frame: "Optional[FrameType]" = None) ->
     GRACEFUL_STOP.set()
 
 
-def run(num_thread=1):
+def run(num_thread: int = 1) -> None:
     """
     Starts up the rucio cache consumer thread
     """