rucio 35.7.0__py3-none-any.whl → 37.0.0rc2__py3-none-any.whl

rucio/daemons/c3po/utils/popularity.py

@@ -1,85 +0,0 @@
-# Copyright European Organization for Nuclear Research (CERN) since 2012
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import logging
-from json import dumps, loads
-from typing import TYPE_CHECKING, Optional
-
-from requests import post
-from requests.auth import HTTPBasicAuth
-
-from rucio.common.config import config_get, config_get_options
-
-if TYPE_CHECKING:
-    from rucio.common.types import InternalScope
-
-ELASTIC_URL = config_get('es-atlas', 'url')
-
-ELASTIC_OPTIONS = config_get_options('es-atlas')
-
-AUTH = None
-if ('username' in ELASTIC_OPTIONS) and ('password' in ELASTIC_OPTIONS):
-    AUTH = HTTPBasicAuth(config_get('es-atlas', 'username'), config_get('es-atlas', 'password'))
-
-if 'ca_cert' in ELASTIC_OPTIONS:
-    ELASTIC_CA_CERT = config_get('es-atlas', 'ca_cert')
-else:
-    ELASTIC_CA_CERT = False
-
-URL = ELASTIC_URL + '/atlas_rucio-popularity-*/_search'
-
-
-def get_popularity(did: tuple['InternalScope', str]) -> Optional[int]:
-    """
-    Query the popularity for a given DID in the ElasticSearch popularity db.
-    """
-    query = {
-        "query": {
-            "bool": {
-                "must": [{
-                    "range": {
-                        "timestamp": {
-                            "gt": "now-7d",
-                            "lt": "now"
-                        }
-                    }
-                }]
-            }
-        },
-        "aggs": {
-            "pop": {"sum": {"field": "ops"}}
-        },
-        "size": 0
-    }
-
-    query['query']['bool']['must'].append({"term": {"scope": did[0].external}})
-    query['query']['bool']['must'].append({"term": {"name": did[1]}})
-
-    logging.debug(query)
-    if AUTH:
-        res = post(URL, data=dumps(query), auth=AUTH, verify=ELASTIC_CA_CERT)
-    else:
-        res = post(URL, data=dumps(query), verify=ELASTIC_CA_CERT)
-
-    if res.status_code != 200:
-        return None
-
-    result = loads(res.text)
-
-    if 'aggregations' in result:
-        if 'pop' in result['aggregations']:
-            if 'value' in result['aggregations']['pop']:
-                return result['aggregations']['pop']['value']
-
-    return None

rucio/daemons/c3po/utils/timeseries.py

@@ -1,89 +0,0 @@
-# Copyright European Organization for Nuclear Research (CERN) since 2012
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""
-Redis time series abstraction
-"""
-
-from time import time
-from typing import TYPE_CHECKING
-
-from redis import StrictRedis
-
-if TYPE_CHECKING:
-    from redis.commands.core import ResponseT
-
-
-class RedisTimeSeries:
-    """
-    Redis time series abstraction
-    """
-
-    def __init__(
-            self,
-            redis_host: str,
-            redis_port: int,
-            window: int,
-            prefix: str
-    ):
-        self._redis = StrictRedis(host=redis_host, port=redis_port)
-        self._prefix = prefix
-        self._window = window * 1000000
-
-    def add_point(
-            self,
-            key: str,
-            value: int
-    ) -> None:
-        """
-        Add a point
-        """
-        r_key = self._prefix + key
-        score = int(time() * 1000000)
-        self._redis.zadd(r_key, score, "%d:%d" % (value, score))  # type: ignore
-
-    def get_series(self, key: str) -> tuple[int]:
-        """
-        Return a time series tuple
-        """
-        r_key = self._prefix + key
-        r_series = self._redis.zrange(r_key, 0, -1)
-        series = []
-        for val in r_series:  # type: ignore
-            values, _ = val.split(':')
-            series.append(int(values))
-
-        return tuple(series)
-
-    def trim(self) -> None:
-        """
-        Trim the time series
-        """
-        now = time()
-        max_score = int(now * 1000000 - self._window)
-        for key in self.get_keys():  # type: ignore
-            self._redis.zremrangebyscore(key, 0, max_score)
-
-    def get_keys(self) -> "ResponseT":
-        """
-        Return matching keys
-        """
-        return self._redis.keys(pattern=self._prefix + "*")
-
-    def delete_keys(self) -> None:
-        """
-        Delete keys
-        """
-        for key in self.get_keys():  # type: ignore
-            self._redis.zremrangebyrank(key, 0, -1)

rucio/rse/protocols/gsiftp.py

@@ -1,92 +0,0 @@
-# Copyright European Organization for Nuclear Research (CERN) since 2012
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import json
-import os
-
-import requests
-
-from rucio.common import exception
-from rucio.rse.protocols import protocol
-
-
-class Default(protocol.RSEProtocol):
-    """ Implementing access to RSEs using gsiftp."""
-
-    def __init__(self, protocol_attr, rse_settings, logger=None):
-        """ Initializes the object with information about the referred RSE.
-
-            :param props: Properties derived from the RSE Repository
-        """
-        super(Default, self).__init__(protocol_attr, rse_settings, logger=logger)
-
-    def connect(self):
-        """
-        Establishes the actual connection to the referred RSE.
-        If we decide to use gfal, init should be done here.
-
-        :raises RSEAccessDenied
-        """
-        pass
-
-    def close(self):
-        """
-        Closes the connection to RSE.
-        """
-        pass
-
-    def get_space_usage(self):
-        """
-        Get RSE space usage information.
-
-        :returns: a list with dict containing 'totalsize' and 'unusedsize'
-
-        :raises ServiceUnavailable: if some generic error occurred in the library.
-        """
-        rse_name = self.rse['rse']
-        dest = '/tmp/rucio-gsiftp-site-size_' + rse_name
-        space_usage_url = ''
-        # url of space usage json, would be nicer to have it in rse_settings
-        agis = requests.get('http://atlas-agis-api.cern.ch/request/ddmendpoint/query/list/?json').json()
-        agis_token = ''
-        for res in agis:
-            if rse_name == res['name']:
-                agis_token = res['token']
-                space_usage_url = res['space_usage_url']
-
-        import gfal2  # pylint: disable=import-error
-        gfal2.set_verbose(gfal2.verbose_level.normal)  # pylint: disable=no-member
-        try:
-            if os.path.exists(dest):
-                os.remove(dest)
-            ctx = gfal2.creat_context()  # pylint: disable=no-member
-            ctx.set_opt_string_list("SRM PLUGIN", "TURL_PROTOCOLS", ["gsiftp", "rfio", "gsidcap", "dcap", "kdcap"])
-            params = ctx.transfer_parameters()
-            params.timeout = 3600
-            ret = ctx.filecopy(params, str(space_usage_url), str('file://' + dest))
-
-            if ret == 0:
-                data_file = open(dest)
-                data = json.load(data_file)
-                data_file.close()
-                if agis_token not in list(data.keys()):
-                    print('ERROR: space usage json has different token as key')
-                else:
-                    totalsize = int(data[agis_token]['total_space'])
-                    used = int(data[agis_token]['used_space'])
-                    unusedsize = totalsize - used
-                    return totalsize, unusedsize
-        except Exception as error:
-            print(error)
-            raise exception.ServiceUnavailable(error)

rucio-35.7.0.data/scripts/rucio-c3po

@@ -1,85 +0,0 @@
-#!python
-# Copyright European Organization for Nuclear Research (CERN) since 2012
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""
-C-3PO is a dynamic data placement daemon.
-"""
-
-import argparse
-import signal
-
-from rucio.common.utils import StoreAndDeprecateWarningAction
-from rucio.daemons.c3po.c3po import run, stop
-
-
-def get_parser():
-    """
-    Returns the argparse parser.
-    """
-    parser = argparse.ArgumentParser(description="The C3PO daemon is responsible for dynamic data placement.")
-    parser.add_argument("--run-once", action="store_true", default=False, help='One iteration only')
-    parser.add_argument("--threads", action="store", default=1, type=int, help='Concurrency control: number of threads')
-    parser.add_argument("--only-workload", action="store_true", default=False, help='Only run the workload collector')
-    parser.add_argument("--dry_run", "--dry-run", new_option_string="--dry-run", action=StoreAndDeprecateWarningAction, default=False, help='Do not create any rules')
-    parser.add_argument("--sampling", action="store_true", default=False, help='In the end flip a to decide to create a rule or not')
-    parser.add_argument("--algorithms", action="store", default="t2_free_space_only_pop_with_network", type=str, help='The placement algorithm or, if in dry_run, a comma separated list of algorithms')
-    parser.add_argument("--datatypes", action="store", default="NTUP,DAOD", type=str, help='Comma separated list of datatype that should trigger the placement')
-    parser.add_argument("--dest_rse_expr", "--dest-rse-expr", new_option_string="--dest-rse-expr", action=StoreAndDeprecateWarningAction, default="type=DATADISK", type=str, help='RSE expression defining the allowed destination RSEs')
-    parser.add_argument("--max_bytes_hour", "--max-bytes-hour", new_option_string="--max-bytes-hour", action=StoreAndDeprecateWarningAction, default=100000000000000, type=int, help='Max number of bytes that c3po is allow to replicate per hour')
-    parser.add_argument("--max_files_hour", "--max-files-hour", new_option_string="--max-files-hour", action=StoreAndDeprecateWarningAction, default=100000, type=int, help='Max number of files that c3po is allow to replicate per hour')
-    parser.add_argument("--max_bytes_hour_rse", "--max-bytes-hour-rse", new_option_string="--max-bytes-hour-rse", action=StoreAndDeprecateWarningAction, default=50000000000000, type=int, help='Max number of bytes that c3po is allow to replicate per hour per rse')  # NOQA: E501
-    parser.add_argument("--max_files_hour_rse", "--max-files-hour-rse", new_option_string="--max-files-hour-rse", action=StoreAndDeprecateWarningAction, default=10000, type=int, help='Max number of files that c3po is allow to replicate per hour prse_rse')
-    parser.add_argument("--min_popularity", "--min-popularity", new_option_string="--min-popularity", action=StoreAndDeprecateWarningAction, default=8, type=int, help='Min number of popularity accesses for a DID in the last 7 days to trigger')
-    parser.add_argument("--min_recent_requests", "--min-recent-requests", new_option_string="--min-recent-requests", action=StoreAndDeprecateWarningAction, default=5, type=int, help='Min number of times a DID has to be requested in the last hour to trigger')  # NOQA: E501
-    parser.add_argument("--max_replicas", "--max-replicas", new_option_string="--max-replicas", action=StoreAndDeprecateWarningAction, default=5, type=int, help='Max number of replicas above which not to trigger anymore')
-    parser.add_argument('--waiting-time-read-free-space', action="store", default=1800, type=int, help='Waiting time for reading free space')
-    parser.add_argument('--waiting-time-read-workload', action="store", default=1800, type=int, help='Waiting time for reading workload')
-    parser.add_argument('--waiting-time-print-workload', action="store", default=600, type=int, help='Waiting time for printing workload')
-    parser.add_argument('--waiting-time-read-dids', action="store", default=60, type=int, help='Waiting time for reading dids')
-    parser.add_argument('--waiting-time-place-replica', action="store", default=100, type=int, help='Waiting time for placing replicas')
-    parser.add_argument('--sleep-time', action="store", default=10, type=int, help='Concurrency control: thread sleep time after each chunk of work')
-    return parser
-
-
-if __name__ == "__main__":
-
-    signal.signal(signal.SIGTERM, stop)
-    parser = get_parser()
-    args = parser.parse_args()
-
-    try:
-        run(once=args.run_once,
-            threads=args.threads,
-            only_workload=args.only_workload,
-            dry_run=args.dry_run,
-            sampling=args.sampling,
-            algorithms=args.algorithms,
-            datatypes=args.datatypes,
-            dest_rse_expr=args.dest_rse_expr,
-            max_bytes_hour=args.max_bytes_hour,
-            max_files_hour=args.max_files_hour,
-            max_bytes_hour_rse=args.max_bytes_hour_rse,
-            max_files_hour_rse=args.max_files_hour_rse,
-            min_popularity=args.min_popularity,
-            min_recent_requests=args.min_recent_requests,
-            max_replicas=args.max_replicas,
-            waiting_time_read_free_space=args.waiting_time_read_free_space,
-            waiting_time_read_workload=args.waiting_time_read_workload,
-            waiting_time_print_workload=args.waiting_time_print_workload,
-            waiting_time_read_dids=args.waiting_time_read_dids,
-            waiting_time_place_replica=args.waiting_time_place_replica,
-            sleep_time=args.sleep_time)
-    except KeyboardInterrupt:
-        stop()

rucio-35.7.0.dist-info/METADATA

@@ -1,72 +0,0 @@
-Metadata-Version: 2.4
-Name: rucio
-Version: 35.7.0
-Summary: Rucio Package
-Home-page: https://rucio.cern.ch/
-Author: Rucio
-Author-email: rucio-dev@cern.ch
-License: Apache License, Version 2.0
-Classifier: Development Status :: 5 - Production/Stable
-Classifier: License :: OSI Approved :: Apache Software License
-Classifier: Intended Audience :: Information Technology
-Classifier: Intended Audience :: System Administrators
-Classifier: Operating System :: POSIX :: Linux
-Classifier: Natural Language :: English
-Classifier: Programming Language :: Python
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Environment :: No Input/Output (Daemon)
-Requires-Python: >=3.9, <4
-License-File: LICENSE
-License-File: AUTHORS.rst
-Requires-Dist: alembic==1.13.2
-Requires-Dist: argcomplete==3.4.0
-Requires-Dist: boto3==1.34.142
-Requires-Dist: dogpile-cache==1.2.2
-Requires-Dist: flask==3.0.3
-Requires-Dist: geoip2==4.8.0
-Requires-Dist: google-auth==2.32.0
-Requires-Dist: jsonschema==4.23.0
-Requires-Dist: oic==1.7.0
-Requires-Dist: paramiko==3.4.0
-Requires-Dist: prometheus-client==0.20.0
-Requires-Dist: pymemcache==4.0.0
-Requires-Dist: python-dateutil==2.9.0.post0
-Requires-Dist: python-magic==0.4.27
-Requires-Dist: redis==5.0.7
-Requires-Dist: requests==2.32.3
-Requires-Dist: sqlalchemy==2.0.31
-Requires-Dist: statsd==4.0.1
-Requires-Dist: stomp-py==8.1.2
-Requires-Dist: tabulate==0.9.0
-Requires-Dist: urllib3==1.26.19
-Provides-Extra: oracle
-Requires-Dist: cx-oracle==8.3.0; extra == "oracle"
-Provides-Extra: mongo
-Requires-Dist: pymongo==4.8.0; extra == "mongo"
-Provides-Extra: postgresql
-Requires-Dist: psycopg2-binary==2.9.9; extra == "postgresql"
-Provides-Extra: mysql
-Requires-Dist: pymysql==1.1.1; extra == "mysql"
-Provides-Extra: kerberos
-Requires-Dist: kerberos==1.3.1; extra == "kerberos"
-Requires-Dist: pykerberos==1.2.4; extra == "kerberos"
-Requires-Dist: requests-kerberos==0.15.0; extra == "kerberos"
-Provides-Extra: globus
-Requires-Dist: globus-sdk==3.41.0; extra == "globus"
-Requires-Dist: pyyaml==6.0.1; extra == "globus"
-Provides-Extra: saml
-Requires-Dist: python3-saml==1.16.0; extra == "saml"
-Provides-Extra: dev
-Requires-Dist: pyyaml==6.0.1; extra == "dev"
-Dynamic: author
-Dynamic: author-email
-Dynamic: classifier
-Dynamic: home-page
-Dynamic: license
-Dynamic: license-file
-Dynamic: provides-extra
-Dynamic: requires-dist
-Dynamic: requires-python
-Dynamic: summary