rucio 32.8.6__py3-none-any.whl

rucio/tests/__init__.py

@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

rucio/tests/common.py

@@ -0,0 +1,244 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import contextlib
+import itertools
+import json
+import os
+import tempfile
+from collections import namedtuple
+from random import choice, choices
+from string import ascii_uppercase, ascii_letters, digits
+from typing import Optional
+
+import pytest
+import requests
+
+from rucio.common.config import config_get, config_get_bool, get_config_dirs
+from rucio.common.utils import generate_uuid as uuid, execute
+
+skip_rse_tests_with_accounts = pytest.mark.skipif(not any(os.path.exists(os.path.join(d, 'rse-accounts.cfg')) for d in get_config_dirs()),
+                                                  reason='fails if no rse-accounts.cfg found')
+skiplimitedsql = pytest.mark.skipif('RDBMS' in os.environ and os.environ['RDBMS'] == 'sqlite',
+                                    reason="does not work in SQLite because of missing features")
+skip_multivo = pytest.mark.skipif('SUITE' in os.environ and os.environ['SUITE'] == 'multi_vo',
+                                  reason="does not work for multiVO")
+skip_non_belleii = pytest.mark.skipif(not ('POLICY' in os.environ and os.environ['POLICY'] == 'belleii'),
+                                      reason="specific belleii tests")
+
+
+def is_influxdb_available() -> bool:
+    """Return True if influxdb is available, else return False."""
+    try:
+        response = requests.get('http://localhost:8086/ping')
+        return response.status_code == 204
+    except requests.exceptions.ConnectionError:
+        print('InfluxDB is not running at localhost:8086')
+        return False
+
+
+def is_elasticsearch_available() -> bool:
+    """Return True if elasticsearch is available, else return False."""
+    try:
+        response = requests.get('http://localhost:9200/')
+        return response.status_code == 200
+    except requests.exceptions.ConnectionError:
+        print('Elasticsearch is not running at localhost:9200')
+        return False
+
+
+skip_missing_elasticsearch_influxdb_in_env = pytest.mark.skipif(not (is_influxdb_available() and is_elasticsearch_available()), reason='influxdb is not available')
+
+
+def get_long_vo() -> str:
+    """ Get the VO name from the config file for testing.
+    Don't map the name to a short version.
+    :returns: VO name string.
+    """
+    vo_name = 'def'
+    if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
+        vo = config_get('client', 'vo', raise_exception=False, default=None)
+        if vo is not None:
+            vo_name = vo
+    return vo_name
+
+
+def account_name_generator() -> str:
+    """ Generate random account name.
+
+    :returns: A random account name
+    """
+    return 'jdoe-' + str(uuid()).lower()[:16]
+
+
+def scope_name_generator() -> str:
+    """ Generate random scope name.
+
+    :returns: A random scope name
+    """
+    return 'mock_' + str(uuid()).lower()[:16]
+
+
+def did_name_generator(did_type: str = 'file', name_prefix: str = '', name_suffix: str = '', path: Optional[str] = None) -> str:
+    """ Generate random did name.
+    :param did_type: A string to create a meaningful did_name depending on the did_type (file, dataset, container)
+    :param name_prefix: String to prefix to the did name
+    :param name_suffix: String to append to the did name
+    :param path: If specified, use the path to generate the did_name
+
+    :returns: A random did name
+    """
+    if os.getenv('POLICY') == 'belleii':
+        if path is not None:
+            return path
+
+        container_path = os.path.join("/belle", name_prefix if name_prefix else "mock", 'cont_%s' % str(uuid()))
+        if did_type == 'container':
+            return container_path
+
+        dataset_path = os.path.join(container_path, 'dataset_%s' % str(uuid()))
+        if did_type == 'dataset':
+            return dataset_path
+
+        file_path = os.path.join(dataset_path, 'file_%s%s' % (str(uuid()), name_suffix))
+        return file_path
+
+    if path is not None:
+        return os.path.basename(path)
+
+    return '%s%s_%s%s' % (name_prefix, did_type, str(uuid()), name_suffix)
+
+
+def rse_name_generator(size: int = 10) -> str:
+    """ Generate random RSE name.
+
+    :returns: A random RSE name
+    """
+    return 'MOCK-' + ''.join(choice(ascii_uppercase) for x in range(size))
+
+
+def rfc2253_dn_generator():
+    """ Generate a random DN in RFC 2253 format.
+
+    :returns: A random DN
+    """
+    random_cn = ''.join(choices(ascii_letters + digits, k=8))
+    random_o = ''.join(choices(ascii_letters + digits, k=8))
+    random_c = ''.join(choices(ascii_letters, k=2))
+    random_dn = "CN={}, O={}, C={}".format(random_cn, random_o, random_c)
+    return random_dn
+
+
+def file_generator(size: int = 2, namelen: int = 10):
+    """ Create a bogus file and returns it's name.
+    :param size: size in bytes
+    :returns: The name of the generated file.
+    """
+    fn = '/tmp/file_' + ''.join(choice(ascii_uppercase) for x in range(namelen))
+    execute('dd if=/dev/urandom of={0} count={1} bs=1'.format(fn, size))
+    return fn
+
+
+def make_temp_file(dir_: str, data: str) -> str:
+    """
+    Creates a temporal file and write `data` on it.
+    :param data: String to be writen on the created file.
+    :returns: Name of the temporal file.
+    """
+    fd, path = tempfile.mkstemp(dir=dir_)
+    with os.fdopen(fd, 'w', encoding='utf-8') as f:
+        f.write(data)
+    return path
+
+
+@contextlib.contextmanager
+def mock_open(module, file_like_object):
+    call_info = {}
+
+    def mocked_open(filename, mode='r'):
+        call_info['filename'] = filename
+        call_info['mode'] = mode
+        file_like_object.close = lambda: None
+        return contextlib.closing(file_like_object)
+
+    setattr(module, 'open', mocked_open)
+    try:
+        yield call_info
+    finally:
+        file_like_object.seek(0)
+        delattr(module, 'open')
+
+
+def print_response(rest_response):
+    print('Status:', rest_response.status)
+    print()
+    nohdrs = True
+    for hdr, val in rest_response.headers.items():
+        if nohdrs:
+            print('Headers:')
+            print('-------')
+            nohdrs = False
+        print('%s: %s' % (hdr, val))
+
+    if not nohdrs:
+        print()
+
+    text = rest_response.get_data(as_text=True)
+    print(text if text else '<no content>')
+
+
+def headers(*iterables):
+    return list(itertools.chain(*iterables))
+
+
+def loginhdr(account, username, password):
+    yield 'X-Rucio-Account', str(account)
+    yield 'X-Rucio-Username', str(username)
+    yield 'X-Rucio-Password', str(password)
+
+
+def auth(token):
+    yield 'X-Rucio-Auth-Token', str(token)
+
+
+def vohdr(vo):
+    if vo:
+        yield 'X-Rucio-VO', str(vo)
+
+
+def hdrdict(dictionary):
+    for key in dictionary:
+        yield str(key), str(dictionary[key])
+
+
+def accept(mimetype):
+    yield 'Accept', mimetype
+
+
+class Mime:
+    """ Enum-type class for mimetypes. """
+    METALINK = 'application/metalink4+xml'
+    JSON = 'application/json'
+    JSON_STREAM = 'application/x-json-stream'
+    BINARY = 'application/octet-stream'
+
+
+def load_test_conf_file(file_name):
+    config_dir = next(filter(lambda d: os.path.exists(os.path.join(d, file_name)), get_config_dirs()))
+    with open(os.path.join(config_dir, file_name)) as f:
+        return json.load(f)
+
+
+RSE_namedtuple = namedtuple('RSE_namedtuple', ['name', 'id'])

rucio/tests/common_server.py

@@ -0,0 +1,132 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import copy
+
+from sqlalchemy import and_, delete, exists, select
+from sqlalchemy.orm import aliased
+
+from rucio.core import config as core_config
+from rucio.core.vo import map_vo
+from rucio.db.sqla import models
+from rucio.db.sqla.session import transactional_session, get_session
+from .common import get_long_vo
+
+# Functions containing server-only includes that can't be included in client tests
+# For each table, get the foreign key constraints from all other tables towards this table.
+INBOUND_FOREIGN_KEYS = {}
+for __table in models.BASE.metadata.tables.values():
+    for __fk in __table.foreign_key_constraints:
+        INBOUND_FOREIGN_KEYS.setdefault(__fk.referred_table, set()).add(__fk)
+
+
+def _dependency_paths(stack, nb_times_in_stack, cur_table):
+    """
+    Generates lists of foreign keys: paths starting at cur_table and
+    navigating the table graph via foreign key constraints.
+
+    For example: As of time of writing, for cur_table = models.ReplicationRule.__table__,
+    it will generate
+        [DATASET_LOCKS_RULE_ID_FK]                            # rule.id <-> dataset_locks.rule_id
+        [LOCKS_RULE_ID_FK]                                    # rule.id <-> locks.rule_id
+        [RULES_CHILD_RULE_ID_FK, DATASET_LOCKS_RULE_ID_FK]    # rule.id <-> rule(alias).child_rule_id, rule(alias).id <-> dataset_locks.rule_id
+        [RULES_CHILD_RULE_ID_FK, LOCKS_RULE_ID_FK]            # rule.id <-> rule(alias).child_rule_id, rule(alias).id <-> locks.rule_id
+        [RULES_CHILD_RULE_ID_FK]                              # rule.id <-> rule(alias).child_rule_id
+    """
+    nb_times_in_stack[cur_table] = nb_times_in_stack.get(cur_table, 0) + 1
+
+    for fk in INBOUND_FOREIGN_KEYS.get(cur_table, []):
+        if nb_times_in_stack.get(fk.table, 0) > 1:
+            # Only allow a table to appear twice in the stack.
+            # This handles recursive constraints (like the one between rules and itself)
+            continue
+        stack.append(fk)
+        yield from _dependency_paths(stack, nb_times_in_stack, fk.table)
+
+    if stack:
+        yield copy.copy(stack)
+        fk = stack.pop()
+        nb_times_in_stack[fk.table] -= 1
+
+
+@transactional_session
+def cleanup_db_deps(model, select_rows_stmt, *, session=None):
+    """
+    Removes rows which have foreign key constraints pointing to rows
+    selected by `select_rows_stmt` in `model`. The deletion is transitive.
+    This implements a behavior similar to "ON DELETE CASCADE", but without
+    removing the initial rows from `model`, only their dependencies.
+    """
+
+    for fk_path in _dependency_paths(stack=[], nb_times_in_stack={}, cur_table=model.__table__):
+        seen_tables = set()
+        referred_table = model.__table__
+        current_table = fk_path[-1].table
+        filters = []
+        for i, fk in enumerate(fk_path):
+            current_table = fk.table
+            if current_table in seen_tables:
+                current_table = aliased(current_table)
+            else:
+                seen_tables.add(current_table)
+
+            filters.append(and_(current_table.columns.get(e.parent.name) == referred_table.columns.get(e.column.name) for e in fk.elements))
+            referred_table = current_table
+
+        if session.bind.dialect.name == 'mysql':
+            stmt = delete(
+                current_table
+            ).where(
+                and_(*filters)
+            ).where(
+                select_rows_stmt
+            )
+        else:
+            stmt = delete(
+                current_table,
+            ).where(
+                exists(
+                    select(
+                        1
+                    ).where(
+                        and_(*filters)
+                    ).where(
+                        select_rows_stmt
+                    )
+                )
+            )
+
+        stmt = stmt.execution_options(
+            synchronize_session=False
+        )
+
+        session.execute(stmt)
+
+
+def reset_config_table():
+    """ Clear the config table and install any default entires needed for the tests.
+    """
+    db_session = get_session()
+    db_session.query(models.Config).delete()
+    db_session.commit()
+    core_config.set("vo-map", "testvo1", "tst")
+    core_config.set("vo-map", "testvo2", "ts2")
+
+
+def get_vo():
+    """ Gets the current short/mapped VO name for testing.
+    Maps the vo name to the short name, if configured.
+    :returns: VO name string.
+    """
+    return map_vo(get_long_vo())

rucio/transfertool/__init__.py

@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.