rucio 32.8.6__py3-none-any.whl

rucio/daemons/c3po/collectors/workload.py

@@ -0,0 +1,110 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+C3PO PanDA workload collector
+"""
+
+import logging
+from json import loads
+from time import time
+
+from requests import get
+
+from rucio.common.config import config_get, config_get_int
+from rucio.daemons.c3po.utils.timeseries import RedisTimeSeries
+
+
+class WorkloadCollector(object):
+    """
+    Collector to retrieve the workload from PanDA. It stores it as a time series in Redis and provides
+    the average and maximum number of running jobs for a sliding window.
+    """
+
+    class __WorkloadCollector(object):
+        """
+        Private class needed implement singleton.
+        """
+        def __init__(self, delete_keys=False):
+            self._avg_jobs = {}
+            self._cur_jobs = {}
+            self._max_jobs = {}
+            self._tms = RedisTimeSeries(config_get('c3po', 'redis_host'), config_get_int('c3po', 'redis_port'), config_get_int('c3po-workload', 'window'), 'jobs_')
+
+            self._request_headers = {"Accept": "application/json", "Content-Type": "application/json"}
+            self._request_url = config_get('c3po-workload', 'panda_url')
+            if delete_keys:
+                self._tms.delete_keys()
+            self.reload_cache()
+
+        def reload_cache(self):
+            self._tms.trim()
+
+            for key in self._tms.get_keys():
+                site = "_".join(key.split('_')[1:])
+                job_series = self._tms.get_series(site)
+                num_jobs = len(job_series)
+                if num_jobs > 0:
+                    self._avg_jobs[site] = sum(job_series) / num_jobs
+                    self._max_jobs[site] = max(job_series)
+                    self._cur_jobs[site] = job_series[-1]
+
+        def collect_workload(self):
+            start = time()
+            resp = get(self._request_url, headers=self._request_headers)
+            logging.debug("PanDA response took %fs" % (time() - start))
+
+            start = time()
+            jobs = loads(resp.text)['jobs']
+            logging.debug("decoding JSON response took %fs" % (time() - start))
+            sites = {}
+
+            start = time()
+            for job in jobs:
+                if job['computingsite'] not in sites:
+                    sites[job['computingsite']] = 0
+                sites[job['computingsite']] += 1
+            for site, jobs in sites.items():
+                self._tms.add_point(site, jobs)
+
+            logging.debug("processing took %fs" % (time() - start))
+            self.reload_cache()
+
+    instance = None
+
+    def __init__(self):
+        if not WorkloadCollector.instance:
+            WorkloadCollector.instance = WorkloadCollector.__WorkloadCollector()
+
+    def get_avg_jobs(self, site):
+        return self.instance._avg_jobs[site]
+
+    def get_max_jobs(self, site):
+        return self.instance._max_jobs[site]
+
+    def get_cur_jobs(self, site):
+        return self.instance._cur_jobs[site]
+
+    def get_sites(self):
+        return list(self.instance._avg_jobs.keys())
+
+    def get_job_info(self, site):
+        return (self.get_cur_jobs(site), self.get_avg_jobs(site), self.get_max_jobs(site))
+
+    def get_series(self, site):
+        return self.instance._tms.get_series(site)
+
+    def collect_workload(self):
+        self.instance.collect_workload()

rucio/daemons/c3po/utils/__init__.py

@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

rucio/daemons/c3po/utils/dataset_cache.py

@@ -0,0 +1,40 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from uuid import uuid4
+
+from rucio.daemons.c3po.utils.timeseries import RedisTimeSeries
+
+
+class DatasetCache(object):
+    """
+    Utility to count the accesses of the datasets during the last day.
+    """
+    def __init__(self, redis_host, redis_port, timeout=1, prefix='did_cache', delete_keys=False):
+        self._prefix = prefix + '_' + str(uuid4()).split('-')[0]
+        self._tms = RedisTimeSeries(redis_host, redis_port, timeout, self._prefix)
+
+        if delete_keys:
+            self._tms.delete_keys()
+
+    def add_did(self, did):
+        self._tms.add_point('{}_{}'.format(did[0].internal, did[1]), 1)
+
+    def get_did(self, did):
+        self._tms.trim()
+
+        series = self._tms.get_series('{}_{}'.format(did[0].internal, did[1]))
+
+        return len(series)

rucio/daemons/c3po/utils/expiring_dataset_cache.py

@@ -0,0 +1,45 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Expiring Dataset Cache
+"""
+from uuid import uuid4
+
+from redis import StrictRedis
+
+
+class ExpiringDatasetCache(object):
+    """
+    Cache with expiring values to keep track of recently created replicas.
+    """
+    def __init__(self, redis_host, redis_port, timeout=1, prefix='expiring_did_cache'):
+        self._redis = StrictRedis(host=redis_host, port=redis_port)
+        self._prefix = prefix + '_' + str(uuid4()).split('-')[0]
+        self._timeout = timeout
+
+    def add_dataset(self, dataset):
+        """ Adds a datasets to cache with lifetime """
+        key = ':'.join((self._prefix, dataset))
+        self._redis.set(key, 1)
+        self._redis.expire(key, self._timeout)
+
+    def check_dataset(self, dataset):
+        """ Checks if dataset is still in cache """
+        key = ':'.join((self._prefix, dataset))
+        if self._redis.get(key) is None:
+            return False
+
+        return True

rucio/daemons/c3po/utils/expiring_list.py

@@ -0,0 +1,63 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Utility classes for C3PO
+"""
+
+from collections import deque
+from threading import Lock, Timer
+
+
+class ExpiringList(object):
+    """
+    Simple list with time based element expiration
+    """
+
+    def __init__(self, timeout=1):
+        self._lock = Lock()
+        self._timeout = timeout
+        self._items = deque()
+
+    def add(self, item):
+        """Add event time
+        """
+        with self._lock:
+            self._items.append(item)
+            Timer(self._timeout, self._expire).start()
+
+    def __len__(self):
+        """
+        Return number of active events
+        """
+        with self._lock:
+            return len(self._items)
+
+    def _expire(self):
+        """
+        Remove any expired events
+        """
+        with self._lock:
+            self._items.popleft()
+
+    def to_set(self):
+        """
+        Return items as a set
+        """
+        return set(self._items)
+
+    def __str__(self):
+        with self._lock:
+            return str(self._items)

rucio/daemons/c3po/utils/popularity.py

@@ -0,0 +1,82 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+from json import dumps, loads
+
+from requests import post
+from requests.auth import HTTPBasicAuth
+
+from rucio.common.config import config_get, config_get_options
+
+ELASTIC_URL = config_get('es-atlas', 'url')
+
+ELASTIC_OPTIONS = config_get_options('es-atlas')
+
+AUTH = None
+if ('username' in ELASTIC_OPTIONS) and ('password' in ELASTIC_OPTIONS):
+    AUTH = HTTPBasicAuth(config_get('es-atlas', 'username'), config_get('es-atlas', 'password'))
+
+if 'ca_cert' in ELASTIC_OPTIONS:
+    ELASTIC_CA_CERT = config_get('es-atlas', 'ca_cert')
+else:
+    ELASTIC_CA_CERT = False
+
+URL = ELASTIC_URL + '/atlas_rucio-popularity-*/_search'
+
+
+def get_popularity(did):
+    """
+    Query the popularity for a given DID in the ElasticSearch popularity db.
+    """
+    query = {
+        "query": {
+            "bool": {
+                "must": [{
+                    "range": {
+                        "timestamp": {
+                            "gt": "now-7d",
+                            "lt": "now"
+                        }
+                    }
+                }]
+            }
+        },
+        "aggs": {
+            "pop": {"sum": {"field": "ops"}}
+        },
+        "size": 0
+    }
+
+    query['query']['bool']['must'].append({"term": {"scope": did[0].external}})
+    query['query']['bool']['must'].append({"term": {"name": did[1]}})
+
+    logging.debug(query)
+    if AUTH:
+        res = post(URL, data=dumps(query), auth=AUTH, verify=ELASTIC_CA_CERT)
+    else:
+        res = post(URL, data=dumps(query), verify=ELASTIC_CA_CERT)
+
+    if res.status_code != 200:
+        return None
+
+    result = loads(res.text)
+
+    if 'aggregations' in result:
+        if 'pop' in result['aggregations']:
+            if 'value' in result['aggregations']['pop']:
+                return result['aggregations']['pop']['value']
+
+    return None

rucio/daemons/c3po/utils/timeseries.py

@@ -0,0 +1,76 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Redis time series abstraction
+"""
+
+from time import time
+
+from redis import StrictRedis
+
+
+class RedisTimeSeries(object):
+    """
+    Redis time series abstraction
+    """
+
+    def __init__(self, redis_host, redis_port, window, prefix):
+        self._redis = StrictRedis(host=redis_host, port=redis_port)
+        self._prefix = prefix
+        self._window = window * 1000000
+
+    def add_point(self, key, value):
+        """
+        Add a point
+        """
+        r_key = self._prefix + key
+        score = int(time() * 1000000)
+        self._redis.zadd(r_key, score, "%d:%d" % (value, score))
+
+    def get_series(self, key):
+        """
+        Return a time series tuple
+        """
+        r_key = self._prefix + key
+        r_series = self._redis.zrange(r_key, 0, -1)
+        series = []
+        for val in r_series:
+            values, _ = val.split(':')
+            series.append(int(values))
+
+        return tuple(series)
+
+    def trim(self):
+        """
+        Trim the time series
+        """
+        now = time()
+        max_score = int(now * 1000000 - self._window)
+        for key in self.get_keys():
+            self._redis.zremrangebyscore(key, 0, max_score)
+
+    def get_keys(self):
+        """
+        Return matching keys
+        """
+        return self._redis.keys(pattern=self._prefix + "*")
+
+    def delete_keys(self):
+        """
+        Delete keys
+        """
+        for key in self.get_keys():
+            self._redis.zremrangebyrank(key, 0, -1)

rucio/daemons/cache/__init__.py

@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

rucio/daemons/cache/consumer.py

@@ -0,0 +1,191 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Cache consumer is a daemon to retrieve rucio cache operation information to synchronize rucio catalog.
+"""
+
+import json
+import logging
+import threading
+import time
+from traceback import format_exc
+from typing import TYPE_CHECKING
+
+import rucio.db.sqla.util
+from rucio.common import exception
+from rucio.common.config import config_get, config_get_int, config_get_bool, config_get_list
+from rucio.common.logging import setup_logging, formatted_logger
+from rucio.common.stomp_utils import StompConnectionManager
+from rucio.common.types import InternalScope
+from rucio.core.monitor import MetricManager
+from rucio.core.rse import get_rse_id
+from rucio.core.volatile_replica import add_volatile_replicas, delete_volatile_replicas
+
+if TYPE_CHECKING:
+    from types import FrameType
+    from typing import Optional
+
+logging.getLogger("stomp").setLevel(logging.CRITICAL)
+
+METRICS = MetricManager(module=__name__)
+GRACEFUL_STOP = threading.Event()
+DAEMON_NAME = 'cache-consumer'
+
+
+class AMQConsumer(object):
+    """
+    class Consumer
+    """
+
+    def __init__(self, broker, conn, logger):
+        """
+        __init__
+        """
+        self.__broker = broker
+        self.__conn = conn
+        self.__logger = logger
+
+    @METRICS.count_it
+    def on_heartbeat_timeout(self):
+        self.__conn.disconnect()
+
+    @METRICS.count_it
+    def on_error(self, frame):
+        """
+        on_error
+        """
+        self.__logger(logging.ERROR, 'Message receive error: [%s] %s' % (self.__broker, frame.body))
+
+    @METRICS.count_it
+    def on_message(self, frame):
+        """
+        on_message
+        """
+        try:
+            msg = json.loads(frame.body)
+            self.__logger(logging.DEBUG, 'Message received: %s ' % msg)
+            if isinstance(msg, dict) and 'operation' in msg.keys():
+                for f in msg['files']:
+                    f['scope'] = InternalScope(f['scope'])
+                if 'rse_id' in msg:
+                    rse_id = msg['rse_id']
+                else:
+                    rse_id = get_rse_id(rse=msg['rse'], vo=msg.get('vo', 'def'))
+
+                rse_vo_str = msg['rse']
+                if 'vo' in msg and msg['vo'] != 'def':
+                    rse_vo_str = '{} on {}'.format(rse_vo_str, msg['vo'])
+                if msg['operation'] == 'add_replicas':
+                    self.__logger(logging.INFO, 'add_replicas to RSE %s: %s ' % (rse_vo_str, str(msg['files'])))
+                    add_volatile_replicas(rse_id=rse_id, replicas=msg['files'])
+                elif msg['operation'] == 'delete_replicas':
+                    self.__logger(logging.INFO, 'delete_replicas to RSE %s: %s ' % (rse_vo_str, str(msg['files'])))
+                    delete_volatile_replicas(rse_id=rse_id, replicas=msg['files'])
+            else:
+                self.__logger(logging.DEBUG, 'Check failed: %s %s '
+                              % (isinstance(msg, dict), 'operation' in msg.keys()))
+        except:
+            self.__logger(logging.ERROR, str(format_exc()))
+
+
+def consumer(id_, num_thread=1):
+    """
+    Main loop to consume messages from the Rucio Cache producer.
+    """
+
+    logger = formatted_logger(logging.log, DAEMON_NAME + ' %s')
+
+    logger(logging.INFO, 'Rucio Cache consumer starting')
+
+    brokers = config_get_list('messaging-cache', 'brokers')
+
+    use_ssl = config_get_bool('messaging-cache', 'use_ssl', default=True, raise_exception=False)
+    if not use_ssl:
+        username = config_get('messaging-cache', 'username')
+        password = config_get('messaging-cache', 'password')
+    destination = config_get('messaging-cache', 'destination')
+    subscription_id = 'rucio-cache-messaging'
+
+    vhost = config_get('messaging-cache', 'broker_virtual_host', raise_exception=False)
+    port = config_get_int('messaging-cache', 'port')
+    reconnect_attempts = config_get_int('messaging-cache', 'reconnect_attempts', default=100)
+    ssl_key_file = config_get('messaging-cache', 'ssl_key_file', raise_exception=False)
+    ssl_cert_file = config_get('messaging-cache', 'ssl_cert_file', raise_exception=False)
+
+    stomp_conn_mngr = StompConnectionManager()
+    conns, _ = stomp_conn_mngr.re_configure(
+        brokers=brokers,
+        port=port,
+        use_ssl=use_ssl,
+        vhost=vhost,
+        reconnect_attempts=reconnect_attempts,
+        ssl_key_file=ssl_key_file,
+        ssl_cert_file=ssl_cert_file,
+        timeout=None,
+        logger=logger
+    )
+
+    logger(logging.INFO, 'consumer started')
+
+    while not GRACEFUL_STOP.is_set():
+        for conn in conns:
+            if not conn.is_connected():
+                host_port = conn.transport._Transport__host_and_ports[0]
+
+                logger(logging.INFO, 'connecting to %s' % host_port[0])
+                METRICS.counter('reconnect.{host}').labels(host=host_port[0]).inc()
+                conn.set_listener('rucio-cache-consumer', AMQConsumer(broker=host_port, conn=conn, logger=logger))
+                if not use_ssl:
+                    conn.connect(username, password)
+                else:
+                    conn.connect()
+
+                conn.subscribe(destination=destination, ack='auto', id=subscription_id)
+        time.sleep(1)
+
+    logger(logging.INFO, 'graceful stop requested')
+    stomp_conn_mngr.disconnect()
+    logger(logging.INFO, 'graceful stop done')
+
+
+def stop(signum: "Optional[int]" = None, frame: "Optional[FrameType]" = None) -> None:
+    """
+    Graceful exit.
+    """
+
+    GRACEFUL_STOP.set()
+
+
+def run(num_thread=1):
+    """
+    Starts up the rucio cache consumer thread
+    """
+    setup_logging(process_name=DAEMON_NAME)
+
+    if rucio.db.sqla.util.is_old_db():
+        raise exception.DatabaseException('Database was not updated, daemon won\'t start')
+
+    logging.info('starting consumer thread')
+    threads = [threading.Thread(target=consumer, kwargs={'id_': i, 'num_thread': num_thread})
+               for i in range(0, num_thread)]
+
+    [t.start() for t in threads]
+
+    logging.info('waiting for interrupts')
+
+    # Interruptible joins require a timeout.
+    while threads[0].is_alive():
+        [t.join(timeout=3.14) for t in threads]