rucio 35.7.0__py3-none-any.whl → 37.0.0__py3-none-any.whl

rucio/daemons/c3po/collectors/agis.py

@@ -1,108 +0,0 @@
-# Copyright European Organization for Nuclear Research (CERN) since 2012
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from json import loads
-from typing import Optional
-
-from requests import get
-
-from rucio.common.config import config_get
-
-
-class MappingCollector:
-    """
-    Provides mappings from PanDA / DDM resources to ATLAS sites and back.
-    """
-    class _MappingCollector:
-        '''
-        _MappingCollector
-        '''
-
-        def __init__(self):
-            '''
-            __init__
-            '''
-            self._fetch_panda_mapping()
-            self._fetch_ddm_mapping()
-
-        def _fetch_panda_mapping(self) -> None:
-            '''
-            _fetch_panda_mapping
-            '''
-            result = get(config_get('c3po-site-mapper', 'panda_url'))
-            data = loads(result.text)
-            self.panda_to_site = {}
-            self.site_to_panda = {}
-
-            for entry in data:
-                self.panda_to_site[entry['panda_resource']] = entry['atlas_site']
-                if entry['atlas_site'] not in self.site_to_panda:
-                    self.site_to_panda[entry['atlas_site']] = []
-                self.site_to_panda[entry['atlas_site']].append(entry['panda_resource'])
-
-        def _fetch_ddm_mapping(self) -> None:
-            '''
-            _fetch_ddm_mapping
-            '''
-            result = get(config_get('c3po-site-mapper', 'ddm_url'))
-            data = loads(result.text)
-            self.site_to_ddm = {}
-            self.ddm_to_site = {}
-
-            for entry in data:
-                self.ddm_to_site[entry['name']] = entry['site']
-                if entry['site'] not in self.site_to_ddm:
-                    self.site_to_ddm[entry['site']] = []
-                self.site_to_ddm[entry['site']].append(entry['name'])
-
-    instance = None
-
-    def __init__(self):
-        '''
-        __init__
-        '''
-        if not MappingCollector.instance:
-            MappingCollector.instance = MappingCollector._MappingCollector()
-
-    def ddm_to_site(self, ddm: str) -> Optional[str]:
-        '''
-        ddm_to_site
-        '''
-        if ddm not in self.instance.ddm_to_site:  # type: ignore
-            return None
-        return self.instance.ddm_to_site[ddm]  # type: ignore
-
-    def panda_to_site(self, panda: str) -> Optional[str]:
-        '''
-        panda_to_site
-        '''
-        if panda not in self.instance.panda_to_site:  # type: ignore
-            return None
-        return self.instance.panda_to_site[panda]  # type: ignore
-
-    def site_to_ddm(self, site: str) -> Optional[str]:
-        '''
-        site_to_ddm
-        '''
-        if site not in self.instance.site_to_ddm:  # type: ignore
-            return None
-        return self.instance.site_to_ddm[site]  # type: ignore
-
-    def site_to_panda(self, site: str) -> Optional[str]:
-        '''
-        site_to_panda
-        '''
-        if site not in self.instance.site_to_panda:  # type: ignore
-            return None
-        return self.instance.site_to_panda[site]  # type: ignore

rucio/daemons/c3po/collectors/free_space.py

@@ -1,81 +0,0 @@
-# Copyright European Organization for Nuclear Research (CERN) since 2012
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""
-Collector to get the SRM free and used information for DATADISK RSEs.
-"""
-
-from typing import TYPE_CHECKING, Optional
-
-from sqlalchemy import and_, select
-
-from rucio.db.sqla.models import RSEAttrAssociation, RSEUsage
-from rucio.db.sqla.session import read_session
-
-if TYPE_CHECKING:
-    from sqlalchemy.orm import Session
-
-
-class FreeSpaceCollector:
-    """
-    Collector to get the SRM free and used information for DATADISK RSEs.
-    """
-    class _FreeSpaceCollector:
-        """
-        Hidden implementation
-        """
-        def __init__(self):
-            self.rses = {}
-
-        @read_session
-        def _collect_free_space(
-            self,
-            *,
-            session: Optional["Session"] = None
-        ) -> None:
-            """
-            Retrieve free space from database
-            """
-            stmt = select(
-                RSEUsage.rse_id,
-                RSEUsage.free,
-                RSEUsage.used
-            ).join(
-                RSEAttrAssociation,
-                RSEAttrAssociation.rse_id == RSEUsage.rse_id
-            ).where(
-                and_(RSEUsage.source == 'storage',
-                     RSEAttrAssociation.key == 'type',
-                     RSEAttrAssociation.value == 'DATADISK'),
-            )
-            for rse_id, free, used in session.execute(stmt).all():  # type: ignore (session could be None)
-                self.rses[rse_id] = {'total': used + free, 'used': used, 'free': free}
-
-    instance = None
-
-    def __init__(self):
-        if not FreeSpaceCollector.instance:
-            FreeSpaceCollector.instance = FreeSpaceCollector._FreeSpaceCollector()
-
-    def collect_free_space(self) -> None:
-        """
-        Execute the free space collector
-        """
-        self.instance._collect_free_space()  # type: ignore
-
-    def get_rse_space(self) -> dict[str, dict[str, int]]:
-        """
-        Return the RSE space
-        """
-        return self.instance.rses  # type: ignore

rucio/daemons/c3po/collectors/jedi_did.py

@@ -1,57 +0,0 @@
-# Copyright European Organization for Nuclear Research (CERN) since 2012
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import logging
-from typing import TYPE_CHECKING, Optional
-
-from rucio.db.sqla.session import read_session
-
-if TYPE_CHECKING:
-    from queue import Queue
-
-    from sqlalchemy.orm import Session
-
-
-class JediDIDCollector:
-    def __init__(self, queue: "Queue"):
-        self.queue = queue
-        self.max_tid = 0
-
-    @read_session
-    def get_dids(
-        self,
-        *,
-        session: Optional["Session"] = None
-    ) -> None:
-        query = """select t.jeditaskid, t.username, t.status, d.datasetname from ATLAS_PANDA.JEDI_TASKS t
-        inner join ATLAS_PANDA.JEDI_DATASETS d
-        on t.jeditaskid = d.jeditaskid
-        where t.creationdate > SYS_EXTRACT_UTC(systimestamp) - 5/(24*60) and t.tasktype = 'anal' and t.prodsourcelabel = 'user'
-        and d.type = 'input'
-        order by d.jeditaskid asc"""
-
-        tasks = session.execute(query)  # type: ignore
-
-        for t in tasks.fetchall():
-            status = t[2]
-            if status == 'running':
-                continue
-            tid = t[0]
-            if tid < self.max_tid:
-                continue
-
-            logging.debug("Received task: " + str(t))
-            did = t[3].split(':')
-            self.queue.put((did[0], did[1]))
-            self.max_tid = tid

rucio/daemons/c3po/collectors/mock_did.py

@@ -1,51 +0,0 @@
-# Copyright European Organization for Nuclear Research (CERN) since 2012
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""
-Mock DID collector
-"""
-
-from secrets import choice
-from typing import TYPE_CHECKING
-
-if TYPE_CHECKING:
-    from multiprocessing import Queue
-
-    from _typeshed import FileDescriptorOrPath
-
-
-class MockDIDCollector:
-    """
-    Simple collector that reads dids from a file. Used to
-    test the interface.
-    """
-    def __init__(self, queue: "Queue"):
-        self._queue = queue
-        self._read_file('/opt/rucio/etc/dids_mc15_13TeV.csv')
-
-    def _read_file(self, infile: "FileDescriptorOrPath") -> None:
-        dids = []
-        with open(infile, 'r') as f:
-            f.readline()
-            for line in f:
-                items = line.strip().split('\t')
-                scope = items[0]
-                name = items[1]
-                dids.append((scope, name))
-
-        self._dids = tuple(dids)
-
-    def get_dids(self) -> None:
-        did = choice(self._dids)
-        self._queue.put(did)

rucio/daemons/c3po/collectors/network_metrics.py

@@ -1,71 +0,0 @@
-# Copyright European Organization for Nuclear Research (CERN) since 2012
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from json import loads
-from typing import Optional
-
-from redis import StrictRedis
-
-from rucio.common.config import config_get, config_get_int
-
-
-class NetworkMetricsCollector:
-    """
-    Collector to get the bandwidth metrics between two sites.
-    """
-
-    def __init__(self):
-        self._r = StrictRedis(host=config_get('c3po-network-metrics', 'redis_host'), port=config_get_int('c3po-network-metrics', 'redis_port'))
-        self._prefix = config_get('c3po-network-metrics', 'prefix')
-
-    def getMbps(
-            self,
-            src: str,
-            type_: str
-    ) -> Optional[dict[str, float]]:
-        pattern = "%s#%s:*" % (self._prefix, src)
-        keys = self._r.keys(pattern=pattern)
-        if len(keys) == 0:
-            return None
-        ret = {}
-
-        vals = self._r.mget(keys)
-
-        for i in range(len(keys)):
-            dst = keys[i].split(':')[1]
-            mbps_all = loads(vals[i]).get('mbps', {}).get(type_, {})
-
-            if '1h' in mbps_all:
-                ret[dst] = float(mbps_all['1h'])
-            else:
-                if '1d' in mbps_all:
-                    ret[dst] = float(mbps_all['1d'])
-                else:
-                    ret[dst] = float(mbps_all.get('1w', 0.0))
-
-        return ret
-
-    def getQueuedFiles(
-            self,
-            src: str,
-            dst: str
-    ) -> int:
-        key = "%s#%s:%s" % (self._prefix, src, dst)
-        activities = loads(self._r.get(key)).get('files', {}).get('queued', {}).get('total', {})
-
-        total = 0
-        for _, values in activities.items():
-            total += values['total']
-
-        return total

rucio/daemons/c3po/collectors/workload.py

@@ -1,112 +0,0 @@
-# Copyright European Organization for Nuclear Research (CERN) since 2012
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""
-C3PO PanDA workload collector
-"""
-
-import logging
-from json import loads
-from time import time
-
-from requests import get
-
-from rucio.common.config import config_get, config_get_int
-from rucio.daemons.c3po.utils.timeseries import RedisTimeSeries
-
-
-class WorkloadCollector:
-    """
-    Collector to retrieve the workload from PanDA. It stores it as a time series in Redis and provides
-    the average and maximum number of running jobs for a sliding window.
-    """
-
-    class __WorkloadCollector:
-        """
-        Private class needed implement singleton.
-        """
-        def __init__(
-                self,
-                delete_keys: bool = False
-        ):
-            self._avg_jobs = {}
-            self._cur_jobs = {}
-            self._max_jobs = {}
-            self._tms = RedisTimeSeries(config_get('c3po', 'redis_host'), config_get_int('c3po', 'redis_port'), config_get_int('c3po-workload', 'window'), 'jobs_')
-
-            self._request_headers = {"Accept": "application/json", "Content-Type": "application/json"}
-            self._request_url = config_get('c3po-workload', 'panda_url')
-            if delete_keys:
-                self._tms.delete_keys()
-            self.reload_cache()
-
-        def reload_cache(self) -> None:
-            self._tms.trim()
-
-            for key in self._tms.get_keys():
-                site = "_".join(key.split('_')[1:])
-                job_series = self._tms.get_series(site)
-                num_jobs = len(job_series)
-                if num_jobs > 0:
-                    self._avg_jobs[site] = sum(job_series) / num_jobs
-                    self._max_jobs[site] = max(job_series)
-                    self._cur_jobs[site] = job_series[-1]
-
-        def collect_workload(self) -> None:
-            start = time()
-            resp = get(self._request_url, headers=self._request_headers)
-            logging.debug("PanDA response took %fs" % (time() - start))
-
-            start = time()
-            jobs = loads(resp.text)['jobs']
-            logging.debug("decoding JSON response took %fs" % (time() - start))
-            sites = {}
-
-            start = time()
-            for job in jobs:
-                if job['computingsite'] not in sites:
-                    sites[job['computingsite']] = 0
-                sites[job['computingsite']] += 1
-            for site, jobs in sites.items():
-                self._tms.add_point(site, jobs)
-
-            logging.debug("processing took %fs" % (time() - start))
-            self.reload_cache()
-
-    instance = None
-
-    def __init__(self):
-        if not WorkloadCollector.instance:
-            WorkloadCollector.instance = WorkloadCollector.__WorkloadCollector()
-
-    def get_avg_jobs(self, site: str) -> float:
-        return self.instance._avg_jobs[site]  # type: ignore
-
-    def get_max_jobs(self, site: str) -> int:
-        return self.instance._max_jobs[site]  # type: ignore
-
-    def get_cur_jobs(self, site: str) -> int:
-        return self.instance._cur_jobs[site]  # type: ignore
-
-    def get_sites(self) -> list[str]:
-        return list(self.instance._avg_jobs.keys())  # type: ignore
-
-    def get_job_info(self, site: str) -> tuple[int, float, int]:
-        return (self.get_cur_jobs(site), self.get_avg_jobs(site), self.get_max_jobs(site))
-
-    def get_series(self, site: str) -> tuple[int]:
-        return self.instance._tms.get_series(site)  # type: ignore
-
-    def collect_workload(self) -> None:
-        self.instance.collect_workload()  # type: ignore

rucio/daemons/c3po/utils/__init__.py

@@ -1,13 +0,0 @@
-# Copyright European Organization for Nuclear Research (CERN) since 2012
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.

rucio/daemons/c3po/utils/dataset_cache.py

@@ -1,50 +0,0 @@
-# Copyright European Organization for Nuclear Research (CERN) since 2012
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from typing import TYPE_CHECKING
-from uuid import uuid4
-
-from rucio.daemons.c3po.utils.timeseries import RedisTimeSeries
-
-if TYPE_CHECKING:
-    from rucio.common.types import InternalScope
-
-
-class DatasetCache:
-    """
-    Utility to count the accesses of the datasets during the last day.
-    """
-    def __init__(
-            self,
-            redis_host: str,
-            redis_port: int,
-            timeout: int = 1,
-            prefix: str = 'did_cache',
-            delete_keys: bool = False
-    ):
-        self._prefix = prefix + '_' + str(uuid4()).split('-')[0]
-        self._tms = RedisTimeSeries(redis_host, redis_port, timeout, self._prefix)
-
-        if delete_keys:
-            self._tms.delete_keys()
-
-    def add_did(self, did: tuple['InternalScope', str]) -> None:
-        self._tms.add_point('{}_{}'.format(did[0].internal, did[1]), 1)
-
-    def get_did(self, did: tuple['InternalScope', str]) -> int:
-        self._tms.trim()
-
-        series = self._tms.get_series('{}_{}'.format(did[0].internal, did[1]))
-
-        return len(series)

rucio/daemons/c3po/utils/expiring_dataset_cache.py

@@ -1,56 +0,0 @@
-# Copyright European Organization for Nuclear Research (CERN) since 2012
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""
-Expiring Dataset Cache
-"""
-from uuid import uuid4
-
-from redis import StrictRedis
-
-
-class ExpiringDatasetCache:
-    """
-    Cache with expiring values to keep track of recently created replicas.
-    """
-    def __init__(
-            self,
-            redis_host: str,
-            redis_port: int,
-            timeout: int = 1,
-            prefix: str = 'expiring_did_cache'
-    ):
-        self._redis = StrictRedis(host=redis_host, port=redis_port)
-        self._prefix = prefix + '_' + str(uuid4()).split('-')[0]
-        self._timeout = timeout
-
-    def add_dataset(
-            self,
-            dataset: str
-    ) -> None:
-        """ Adds a datasets to cache with lifetime """
-        key = ':'.join((self._prefix, dataset))
-        self._redis.set(key, 1)
-        self._redis.expire(key, self._timeout)
-
-    def check_dataset(
-            self,
-            dataset: str
-    ) -> bool:
-        """ Checks if dataset is still in cache """
-        key = ':'.join((self._prefix, dataset))
-        if self._redis.get(key) is None:
-            return False
-
-        return True

rucio/daemons/c3po/utils/expiring_list.py

@@ -1,62 +0,0 @@
-# Copyright European Organization for Nuclear Research (CERN) since 2012
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""
-Utility classes for C3PO
-"""
-
-from collections import deque
-from threading import Lock, Timer
-
-
-class ExpiringList:
-    """
-    Simple list with time based element expiration
-    """
-
-    def __init__(self, timeout: int = 1):
-        self._lock = Lock()
-        self._timeout = timeout
-        self._items = deque()
-
-    def add(self, item):
-        """Add event time
-        """
-        with self._lock:
-            self._items.append(item)
-            Timer(self._timeout, self._expire).start()
-
-    def __len__(self) -> int:
-        """
-        Return number of active events
-        """
-        with self._lock:
-            return len(self._items)
-
-    def _expire(self) -> None:
-        """
-        Remove any expired events
-        """
-        with self._lock:
-            self._items.popleft()
-
-    def to_set(self) -> set:
-        """
-        Return items as a set
-        """
-        return set(self._items)
-
-    def __str__(self) -> str:
-        with self._lock:
-            return str(self._items)