rucio 35.7.0__py3-none-any.whl

rucio/daemons/c3po/collectors/network_metrics.py

@@ -0,0 +1,71 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from json import loads
+from typing import Optional
+
+from redis import StrictRedis
+
+from rucio.common.config import config_get, config_get_int
+
+
+class NetworkMetricsCollector:
+    """
+    Collector to get the bandwidth metrics between two sites.
+    """
+
+    def __init__(self):
+        self._r = StrictRedis(host=config_get('c3po-network-metrics', 'redis_host'), port=config_get_int('c3po-network-metrics', 'redis_port'))
+        self._prefix = config_get('c3po-network-metrics', 'prefix')
+
+    def getMbps(
+            self,
+            src: str,
+            type_: str
+    ) -> Optional[dict[str, float]]:
+        pattern = "%s#%s:*" % (self._prefix, src)
+        keys = self._r.keys(pattern=pattern)
+        if len(keys) == 0:
+            return None
+        ret = {}
+
+        vals = self._r.mget(keys)
+
+        for i in range(len(keys)):
+            dst = keys[i].split(':')[1]
+            mbps_all = loads(vals[i]).get('mbps', {}).get(type_, {})
+
+            if '1h' in mbps_all:
+                ret[dst] = float(mbps_all['1h'])
+            else:
+                if '1d' in mbps_all:
+                    ret[dst] = float(mbps_all['1d'])
+                else:
+                    ret[dst] = float(mbps_all.get('1w', 0.0))
+
+        return ret
+
+    def getQueuedFiles(
+            self,
+            src: str,
+            dst: str
+    ) -> int:
+        key = "%s#%s:%s" % (self._prefix, src, dst)
+        activities = loads(self._r.get(key)).get('files', {}).get('queued', {}).get('total', {})
+
+        total = 0
+        for _, values in activities.items():
+            total += values['total']
+
+        return total

rucio/daemons/c3po/collectors/workload.py

@@ -0,0 +1,112 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+C3PO PanDA workload collector
+"""
+
+import logging
+from json import loads
+from time import time
+
+from requests import get
+
+from rucio.common.config import config_get, config_get_int
+from rucio.daemons.c3po.utils.timeseries import RedisTimeSeries
+
+
+class WorkloadCollector:
+    """
+    Collector to retrieve the workload from PanDA. It stores it as a time series in Redis and provides
+    the average and maximum number of running jobs for a sliding window.
+    """
+
+    class __WorkloadCollector:
+        """
+        Private class needed implement singleton.
+        """
+        def __init__(
+                self,
+                delete_keys: bool = False
+        ):
+            self._avg_jobs = {}
+            self._cur_jobs = {}
+            self._max_jobs = {}
+            self._tms = RedisTimeSeries(config_get('c3po', 'redis_host'), config_get_int('c3po', 'redis_port'), config_get_int('c3po-workload', 'window'), 'jobs_')
+
+            self._request_headers = {"Accept": "application/json", "Content-Type": "application/json"}
+            self._request_url = config_get('c3po-workload', 'panda_url')
+            if delete_keys:
+                self._tms.delete_keys()
+            self.reload_cache()
+
+        def reload_cache(self) -> None:
+            self._tms.trim()
+
+            for key in self._tms.get_keys():
+                site = "_".join(key.split('_')[1:])
+                job_series = self._tms.get_series(site)
+                num_jobs = len(job_series)
+                if num_jobs > 0:
+                    self._avg_jobs[site] = sum(job_series) / num_jobs
+                    self._max_jobs[site] = max(job_series)
+                    self._cur_jobs[site] = job_series[-1]
+
+        def collect_workload(self) -> None:
+            start = time()
+            resp = get(self._request_url, headers=self._request_headers)
+            logging.debug("PanDA response took %fs" % (time() - start))
+
+            start = time()
+            jobs = loads(resp.text)['jobs']
+            logging.debug("decoding JSON response took %fs" % (time() - start))
+            sites = {}
+
+            start = time()
+            for job in jobs:
+                if job['computingsite'] not in sites:
+                    sites[job['computingsite']] = 0
+                sites[job['computingsite']] += 1
+            for site, jobs in sites.items():
+                self._tms.add_point(site, jobs)
+
+            logging.debug("processing took %fs" % (time() - start))
+            self.reload_cache()
+
+    instance = None
+
+    def __init__(self):
+        if not WorkloadCollector.instance:
+            WorkloadCollector.instance = WorkloadCollector.__WorkloadCollector()
+
+    def get_avg_jobs(self, site: str) -> float:
+        return self.instance._avg_jobs[site]  # type: ignore
+
+    def get_max_jobs(self, site: str) -> int:
+        return self.instance._max_jobs[site]  # type: ignore
+
+    def get_cur_jobs(self, site: str) -> int:
+        return self.instance._cur_jobs[site]  # type: ignore
+
+    def get_sites(self) -> list[str]:
+        return list(self.instance._avg_jobs.keys())  # type: ignore
+
+    def get_job_info(self, site: str) -> tuple[int, float, int]:
+        return (self.get_cur_jobs(site), self.get_avg_jobs(site), self.get_max_jobs(site))
+
+    def get_series(self, site: str) -> tuple[int]:
+        return self.instance._tms.get_series(site)  # type: ignore
+
+    def collect_workload(self) -> None:
+        self.instance.collect_workload()  # type: ignore

rucio/daemons/c3po/utils/__init__.py

@@ -0,0 +1,13 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

rucio/daemons/c3po/utils/dataset_cache.py

@@ -0,0 +1,50 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import TYPE_CHECKING
+from uuid import uuid4
+
+from rucio.daemons.c3po.utils.timeseries import RedisTimeSeries
+
+if TYPE_CHECKING:
+    from rucio.common.types import InternalScope
+
+
+class DatasetCache:
+    """
+    Utility to count the accesses of the datasets during the last day.
+    """
+    def __init__(
+            self,
+            redis_host: str,
+            redis_port: int,
+            timeout: int = 1,
+            prefix: str = 'did_cache',
+            delete_keys: bool = False
+    ):
+        self._prefix = prefix + '_' + str(uuid4()).split('-')[0]
+        self._tms = RedisTimeSeries(redis_host, redis_port, timeout, self._prefix)
+
+        if delete_keys:
+            self._tms.delete_keys()
+
+    def add_did(self, did: tuple['InternalScope', str]) -> None:
+        self._tms.add_point('{}_{}'.format(did[0].internal, did[1]), 1)
+
+    def get_did(self, did: tuple['InternalScope', str]) -> int:
+        self._tms.trim()
+
+        series = self._tms.get_series('{}_{}'.format(did[0].internal, did[1]))
+
+        return len(series)

rucio/daemons/c3po/utils/expiring_dataset_cache.py

@@ -0,0 +1,56 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Expiring Dataset Cache
+"""
+from uuid import uuid4
+
+from redis import StrictRedis
+
+
+class ExpiringDatasetCache:
+    """
+    Cache with expiring values to keep track of recently created replicas.
+    """
+    def __init__(
+            self,
+            redis_host: str,
+            redis_port: int,
+            timeout: int = 1,
+            prefix: str = 'expiring_did_cache'
+    ):
+        self._redis = StrictRedis(host=redis_host, port=redis_port)
+        self._prefix = prefix + '_' + str(uuid4()).split('-')[0]
+        self._timeout = timeout
+
+    def add_dataset(
+            self,
+            dataset: str
+    ) -> None:
+        """ Adds a datasets to cache with lifetime """
+        key = ':'.join((self._prefix, dataset))
+        self._redis.set(key, 1)
+        self._redis.expire(key, self._timeout)
+
+    def check_dataset(
+            self,
+            dataset: str
+    ) -> bool:
+        """ Checks if dataset is still in cache """
+        key = ':'.join((self._prefix, dataset))
+        if self._redis.get(key) is None:
+            return False
+
+        return True

rucio/daemons/c3po/utils/expiring_list.py

@@ -0,0 +1,62 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Utility classes for C3PO
+"""
+
+from collections import deque
+from threading import Lock, Timer
+
+
+class ExpiringList:
+    """
+    Simple list with time based element expiration
+    """
+
+    def __init__(self, timeout: int = 1):
+        self._lock = Lock()
+        self._timeout = timeout
+        self._items = deque()
+
+    def add(self, item):
+        """Add event time
+        """
+        with self._lock:
+            self._items.append(item)
+            Timer(self._timeout, self._expire).start()
+
+    def __len__(self) -> int:
+        """
+        Return number of active events
+        """
+        with self._lock:
+            return len(self._items)
+
+    def _expire(self) -> None:
+        """
+        Remove any expired events
+        """
+        with self._lock:
+            self._items.popleft()
+
+    def to_set(self) -> set:
+        """
+        Return items as a set
+        """
+        return set(self._items)
+
+    def __str__(self) -> str:
+        with self._lock:
+            return str(self._items)

rucio/daemons/c3po/utils/popularity.py

@@ -0,0 +1,85 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+from json import dumps, loads
+from typing import TYPE_CHECKING, Optional
+
+from requests import post
+from requests.auth import HTTPBasicAuth
+
+from rucio.common.config import config_get, config_get_options
+
+if TYPE_CHECKING:
+    from rucio.common.types import InternalScope
+
+ELASTIC_URL = config_get('es-atlas', 'url')
+
+ELASTIC_OPTIONS = config_get_options('es-atlas')
+
+AUTH = None
+if ('username' in ELASTIC_OPTIONS) and ('password' in ELASTIC_OPTIONS):
+    AUTH = HTTPBasicAuth(config_get('es-atlas', 'username'), config_get('es-atlas', 'password'))
+
+if 'ca_cert' in ELASTIC_OPTIONS:
+    ELASTIC_CA_CERT = config_get('es-atlas', 'ca_cert')
+else:
+    ELASTIC_CA_CERT = False
+
+URL = ELASTIC_URL + '/atlas_rucio-popularity-*/_search'
+
+
+def get_popularity(did: tuple['InternalScope', str]) -> Optional[int]:
+    """
+    Query the popularity for a given DID in the ElasticSearch popularity db.
+    """
+    query = {
+        "query": {
+            "bool": {
+                "must": [{
+                    "range": {
+                        "timestamp": {
+                            "gt": "now-7d",
+                            "lt": "now"
+                        }
+                    }
+                }]
+            }
+        },
+        "aggs": {
+            "pop": {"sum": {"field": "ops"}}
+        },
+        "size": 0
+    }
+
+    query['query']['bool']['must'].append({"term": {"scope": did[0].external}})
+    query['query']['bool']['must'].append({"term": {"name": did[1]}})
+
+    logging.debug(query)
+    if AUTH:
+        res = post(URL, data=dumps(query), auth=AUTH, verify=ELASTIC_CA_CERT)
+    else:
+        res = post(URL, data=dumps(query), verify=ELASTIC_CA_CERT)
+
+    if res.status_code != 200:
+        return None
+
+    result = loads(res.text)
+
+    if 'aggregations' in result:
+        if 'pop' in result['aggregations']:
+            if 'value' in result['aggregations']['pop']:
+                return result['aggregations']['pop']['value']
+
+    return None

rucio/daemons/c3po/utils/timeseries.py

@@ -0,0 +1,89 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Redis time series abstraction
+"""
+
+from time import time
+from typing import TYPE_CHECKING
+
+from redis import StrictRedis
+
+if TYPE_CHECKING:
+    from redis.commands.core import ResponseT
+
+
+class RedisTimeSeries:
+    """
+    Redis time series abstraction
+    """
+
+    def __init__(
+            self,
+            redis_host: str,
+            redis_port: int,
+            window: int,
+            prefix: str
+    ):
+        self._redis = StrictRedis(host=redis_host, port=redis_port)
+        self._prefix = prefix
+        self._window = window * 1000000
+
+    def add_point(
+            self,
+            key: str,
+            value: int
+    ) -> None:
+        """
+        Add a point
+        """
+        r_key = self._prefix + key
+        score = int(time() * 1000000)
+        self._redis.zadd(r_key, score, "%d:%d" % (value, score))  # type: ignore
+
+    def get_series(self, key: str) -> tuple[int]:
+        """
+        Return a time series tuple
+        """
+        r_key = self._prefix + key
+        r_series = self._redis.zrange(r_key, 0, -1)
+        series = []
+        for val in r_series:  # type: ignore
+            values, _ = val.split(':')
+            series.append(int(values))
+
+        return tuple(series)
+
+    def trim(self) -> None:
+        """
+        Trim the time series
+        """
+        now = time()
+        max_score = int(now * 1000000 - self._window)
+        for key in self.get_keys():  # type: ignore
+            self._redis.zremrangebyscore(key, 0, max_score)
+
+    def get_keys(self) -> "ResponseT":
+        """
+        Return matching keys
+        """
+        return self._redis.keys(pattern=self._prefix + "*")
+
+    def delete_keys(self) -> None:
+        """
+        Delete keys
+        """
+        for key in self.get_keys():  # type: ignore
+            self._redis.zremrangebyrank(key, 0, -1)

rucio/daemons/cache/__init__.py

@@ -0,0 +1,13 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.