rucio-clients 37.0.0rc1__py3-none-any.whl

rucio/common/plugins.py

@@ -0,0 +1,185 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import importlib
+import logging
+import os
+from configparser import NoOptionError, NoSectionError
+from typing import TYPE_CHECKING, Any, TypeVar
+
+from packaging.specifiers import SpecifierSet
+
+from rucio.common import config
+from rucio.common.client import get_client_vo, is_client
+from rucio.common.exception import InvalidAlgorithmName, PolicyPackageIsNotVersioned, PolicyPackageVersionError
+from rucio.version import current_version
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+    from rucio.common.types import LoggerFunction
+
+PolicyPackageAlgorithmsT = TypeVar('PolicyPackageAlgorithmsT', bound='PolicyPackageAlgorithms')
+
+
+def check_policy_package_version(package: str, logger: 'LoggerFunction' = logging.log) -> None:
+
+    '''
+    Checks that the Rucio version supported by the policy package is compatible
+    with this version. Raises an exception if not.
+    :param package: the fully qualified name of the policy package
+    '''
+    try:
+        supported_versionset = _get_supported_versions_from_policy_package(package)
+    except ImportError:
+        logger(logging.DEBUG, 'Policy package %s not found' % package)
+        return
+    except PolicyPackageIsNotVersioned:
+        logger(logging.DEBUG, 'Policy package %s does not include information about which Rucio versions it supports' % package)
+        return
+
+    rucio_version = current_version()
+    if rucio_version not in supported_versionset:
+        raise PolicyPackageVersionError(rucio_version=rucio_version, supported_versionset=str(supported_versionset), package=package)
+
+
+def _get_supported_versions_from_policy_package(package: str) -> SpecifierSet:
+    try:
+        module = importlib.import_module(package)
+    except ImportError as e:
+        raise e
+
+    if not hasattr(module, 'SUPPORTED_VERSION'):
+        raise PolicyPackageIsNotVersioned(package)
+
+    supported_versionset = module.SUPPORTED_VERSION
+
+    if isinstance(supported_versionset, list):
+        supported_versionset = ','.join(supported_versionset)
+
+    return SpecifierSet(supported_versionset)
+
+
+class PolicyPackageAlgorithms:
+    """
+    Base class for Rucio Policy Package Algorithms
+
+    ALGORITHMS is a dict where:
+        - the key is the algorithm type
+        - the value is a dictionary of algorithm names and their callables
+    """
+    _ALGORITHMS: dict[str, dict[str, 'Callable[..., Any]']] = {}
+    _loaded_policy_modules = False
+
+    def __init__(self) -> None:
+        if not self._loaded_policy_modules:
+            self._register_all_policy_package_algorithms()
+            self._loaded_policy_modules = True
+
+    @classmethod
+    def _get_one_algorithm(cls: type[PolicyPackageAlgorithmsT], algorithm_type: str, name: str) -> 'Callable[..., Any]':
+        """
+        Get the algorithm from the dictionary of algorithms
+        """
+        return cls._ALGORITHMS[algorithm_type][name]
+
+    @classmethod
+    def _get_algorithms(cls: type[PolicyPackageAlgorithmsT], algorithm_type: str) -> dict[str, 'Callable[..., Any]']:
+        """
+        Get the dictionary of algorithms for a given type
+        """
+        return cls._ALGORITHMS[algorithm_type]
+
+    @classmethod
+    def _register(
+            cls: type[PolicyPackageAlgorithmsT],
+            algorithm_type: str, algorithm_dict: dict[str, 'Callable[..., Any]']) -> None:
+        """
+        Provided a dictionary of callable function,
+        and the associated algorithm type,
+        register it as one of the valid algorithms.
+        """
+        if algorithm_type in cls._ALGORITHMS:
+            cls._ALGORITHMS[algorithm_type].update(algorithm_dict)
+        else:
+            cls._ALGORITHMS[algorithm_type] = algorithm_dict
+
+    @classmethod
+    def _supports(cls: type[PolicyPackageAlgorithmsT], algorithm_type: str, name: str) -> bool:
+        """
+        Check if a algorithm is supported by the plugin
+        """
+        return name in cls._ALGORITHMS.get(algorithm_type, {})
+
+    @classmethod
+    def _register_all_policy_package_algorithms(cls: type[PolicyPackageAlgorithmsT]) -> None:
+        '''
+        Loads all the algorithms of a given type from the policy package(s) and registers them
+        :param algorithm_type: the type of algorithm to register (e.g. 'lfn2pfn')
+        :param dictionary: the dictionary to register them in
+        :param vo: the name of the relevant VO (None for single VO)
+        '''
+        try:
+            multivo = config.config_get_bool('common', 'multi_vo')
+        except (NoOptionError, NoSectionError):
+            multivo = False
+        if not multivo:
+            # single policy package
+            cls._try_importing_policy()
+        else:
+            # on client, only register algorithms for selected VO
+            if is_client():
+                vo = get_client_vo()
+                cls._try_importing_policy(vo)
+            # on server, list all VOs and register their algorithms
+            else:
+                from rucio.core.vo import list_vos
+                # policy package per VO
+                vos = list_vos()
+                for vo in vos:
+                    cls._try_importing_policy(vo['vo'])
+
+    @classmethod
+    def _try_importing_policy(cls: type[PolicyPackageAlgorithmsT], vo: str = "") -> None:
+        try:
+            # import from utils here to avoid circular import
+
+            env_name = 'RUCIO_POLICY_PACKAGE' + ('' if not vo else '_' + vo.upper())
+            package = getattr(os.environ, env_name, "")
+            if not package:
+                package = str(config.config_get('policy', 'package' + ('' if not vo else '-' + vo)))
+
+            check_policy_package_version(package)
+            module = importlib.import_module(package)
+
+            if hasattr(module, 'get_algorithms'):
+                all_algorithms = module.get_algorithms()
+
+                # for backward compatibility, rename 'surl' to 'non_deterministic_pfn' here
+                if 'surl' in all_algorithms:
+                    all_algorithms['non_deterministic_pfn'] = all_algorithms['surl']
+
+                # check that the names are correctly prefixed for multi-VO
+                if vo:
+                    for _, algorithms in all_algorithms.items():
+                        for k in algorithms.keys():
+                            if not k.lower().startswith(vo.lower()):
+                                raise InvalidAlgorithmName(k, vo)
+
+                # Updates the dictionary with the algorithms from the policy package
+                for algorithm_type, algorithm_dict in all_algorithms.items():
+                    cls._register(algorithm_type, algorithm_dict)
+
+        except (NoOptionError, NoSectionError, ImportError):
+            pass

rucio/common/policy.py

@@ -0,0 +1,93 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import logging
+import os
+from configparser import NoOptionError, NoSectionError
+from functools import wraps
+from typing import TYPE_CHECKING, Any
+
+from dogpile.cache import make_region
+from dogpile.cache.api import NoValue
+
+from rucio.common.config import config_get
+from rucio.common.exception import UndefinedPolicy
+
+if TYPE_CHECKING:
+    from rucio.common.types import LoggerFunction
+
+REGION = make_region().configure('dogpile.cache.memory',
+                                 expiration_time=900)
+
+
+def get_policy(logger: 'LoggerFunction' = logging.log) -> str:
+    policy = REGION.get('policy')
+    if isinstance(policy, NoValue):
+        try:
+            policy = config_get('policy', 'permission')
+        except (NoOptionError, NoSectionError):
+            try:
+                policy = config_get('permission', 'policy')
+            except (NoOptionError, NoSectionError):
+                policy = 'def'
+                logger(logging.WARNING, "Policy not specified, falling back to 'def'")
+        policy = os.environ.get('POLICY', policy)
+        REGION.set('policy', policy)
+    return policy
+
+
+def get_scratchdisk_lifetime() -> int:
+    scratchdisk_lifetime = REGION.get('scratchdisk_lifetime')
+    if isinstance(scratchdisk_lifetime, NoValue):
+        try:
+            scratchdisk_lifetime = config_get('policy', 'scratchdisk_lifetime')
+            scratchdisk_lifetime = int(scratchdisk_lifetime)
+        except (NoOptionError, NoSectionError, ValueError):
+            scratchdisk_lifetime = 14
+        REGION.set('scratchdisk_lifetime', scratchdisk_lifetime)
+    return scratchdisk_lifetime
+
+
+def get_lifetime_policy() -> dict[str, list[dict[str, Any]]]:
+    lifetime_dict = REGION.get('lifetime_dict')
+    if isinstance(lifetime_dict, NoValue):
+        lifetime_dict = {'data': [], 'mc': [], 'valid': [], 'other': []}
+        lifetime_dir = '/opt/rucio/etc/policies'
+        try:
+            lifetime_dir = config_get('lifetime', 'directory')
+        except (NoSectionError, NoOptionError):
+            pass
+        for dtype in ['data', 'mc', 'valid', 'other']:
+            input_file_name = '%s/config_%s.json' % (lifetime_dir, dtype)
+            if os.path.isfile(input_file_name):
+                with open(input_file_name, 'r') as input_file:
+                    lifetime_dict[dtype] = json.load(input_file)
+        REGION.set('lifetime_dict', lifetime_dict)
+    return lifetime_dict
+
+
+def policy_filter(function):
+    mapping = {'atlas': ['get_scratch_policy', 'archive_localgroupdisk_datasets']}
+    policy = get_policy()
+    if policy in mapping and function.__name__ in mapping[policy]:
+        @wraps(function)
+        def new_funct(*args, **kwargs):
+            return function(*args, **kwargs)
+        return new_funct
+
+    @wraps(function)
+    def raise_funct(*args, **kwargs):
+        raise UndefinedPolicy
+    return raise_funct

rucio/common/schema/__init__.py

@@ -0,0 +1,200 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import functools
+import importlib
+import logging
+from configparser import NoOptionError, NoSectionError
+from os import environ
+from typing import TYPE_CHECKING, Any
+
+from jsonschema import ValidationError, validate
+
+from rucio.common import config, exception
+from rucio.common.plugins import check_policy_package_version
+
+if TYPE_CHECKING:
+    from types import ModuleType
+
+LOGGER = logging.getLogger('policy')
+
+# dictionary of schema modules for each VO
+schema_modules: dict[str, "ModuleType"] = {}
+
+# list of unique SCOPE_NAME_REGEXP values from all schemas
+scope_name_regexps: list[str] = []
+
+
+# cached function to check for multivo
+@functools.cache
+def _is_multivo():
+    try:
+        return config.config_get_bool('common', 'multi_vo', check_config_table=False)
+    except (NoOptionError, NoSectionError):
+        return False
+
+
+# cached function to get generic schema module
+@functools.cache
+def _get_generic_schema_module():
+    generic_fallback = 'generic_multi_vo' if _is_multivo() else 'generic'
+    return importlib.import_module('rucio.common.schema.' + generic_fallback)
+
+
+# multi-VO version loads schema per-VO on demand
+# we can't get a list of VOs here because the database might not
+# be available as this is imported during the bootstrapping process
+if not _is_multivo():
+    GENERIC_FALLBACK = 'generic'
+
+    if config.config_has_section('policy'):
+        try:
+            if 'RUCIO_POLICY_PACKAGE' in environ:
+                policy = environ['RUCIO_POLICY_PACKAGE']
+            else:
+                policy = config.config_get('policy', 'package', check_config_table=False)
+            check_policy_package_version(policy)
+            policy = policy + ".schema"
+        except (NoOptionError, NoSectionError):
+            # fall back to old system for now
+            try:
+                policy = config.config_get('policy', 'schema', check_config_table=False)
+            except (NoOptionError, NoSectionError):
+                policy = GENERIC_FALLBACK
+            policy = 'rucio.common.schema.' + policy.lower()
+    else:
+        policy = 'rucio.common.schema.' + GENERIC_FALLBACK.lower()
+
+    try:
+        module = importlib.import_module(policy)
+    except ModuleNotFoundError:
+        # if policy package does not contain schema module, load fallback module instead
+        # this allows a policy package to omit modules that do not need customisation
+        try:
+            LOGGER.warning('Unable to load schema module %s from policy package, falling back to %s'
+                           % (policy, GENERIC_FALLBACK))
+            policy = 'rucio.common.schema.' + GENERIC_FALLBACK.lower()
+            module = importlib.import_module(policy)
+        except ModuleNotFoundError:
+            raise exception.PolicyPackageNotFound(policy)
+        except ImportError:
+            raise exception.ErrorLoadingPolicyPackage(policy)
+    except ImportError:
+        raise exception.ErrorLoadingPolicyPackage(policy)
+
+    schema_modules["def"] = module
+    if hasattr(module, 'SCOPE_NAME_REGEXP'):
+        scope_name_regexps.append(module.SCOPE_NAME_REGEXP)
+
+
+def load_schema_for_vo(vo: str) -> None:
+    generic_fallback = 'generic_multi_vo'
+    if config.config_has_section('policy'):
+        try:
+            env_name = 'RUCIO_POLICY_PACKAGE_' + vo.upper()
+            if env_name in environ:
+                policy = environ[env_name]
+            else:
+                policy = config.config_get('policy', 'package-' + vo, check_config_table=False)
+            check_policy_package_version(policy)
+            policy = policy + ".schema"
+        except (NoOptionError, NoSectionError):
+            # fall back to old system for now
+            try:
+                policy = config.config_get('policy', 'schema', check_config_table=False)
+            except (NoOptionError, NoSectionError):
+                policy = generic_fallback
+            policy = 'rucio.common.schema.' + policy.lower()
+    else:
+        policy = 'rucio.common.schema.' + generic_fallback.lower()
+
+    try:
+        module = importlib.import_module(policy)
+    except ModuleNotFoundError:
+        # if policy package does not contain schema module, load fallback module instead
+        # this allows a policy package to omit modules that do not need customisation
+        try:
+            LOGGER.warning('Unable to load schema module %s from policy package, falling back to %s'
+                           % (policy, generic_fallback))
+            policy = 'rucio.common.schema.' + generic_fallback.lower()
+            module = importlib.import_module(policy)
+        except ModuleNotFoundError:
+            raise exception.PolicyPackageNotFound(policy)
+        except ImportError:
+            raise exception.ErrorLoadingPolicyPackage(policy)
+    except ImportError:
+        raise exception.ErrorLoadingPolicyPackage(policy)
+
+    schema_modules[vo] = module
+
+
+def validate_schema(name: str, obj: Any, vo: str = 'def') -> None:
+    if obj:
+        if vo not in schema_modules:
+            load_schema_for_vo(vo)
+        if hasattr(schema_modules[vo], 'SCHEMAS') and name in schema_modules[vo].SCHEMAS:
+            schema = schema_modules[vo].SCHEMAS.get(name, {})
+        else:
+            # if schema not available in VO module, fall back to generic module
+            schema = _get_generic_schema_module().SCHEMAS.get(name, {})
+        try:
+            validate(obj, schema)
+        except ValidationError as error:  # NOQA: F841
+            raise exception.InvalidObject(f'Problem validating {name}: {error}')
+
+
+def get_schema_value(key: str, vo: str = 'def') -> Any:
+    if vo not in schema_modules:
+        load_schema_for_vo(vo)
+    if not hasattr(schema_modules[vo], key):
+        return getattr(_get_generic_schema_module(), key)
+    return getattr(schema_modules[vo], key)
+
+
+def get_scope_name_regexps() -> list[str]:
+    """ returns a list of all unique SCOPE_NAME_REGEXPs from all schemas """
+
+    if len(scope_name_regexps) == 0:
+        # load schemas for all VOs here and add unique scope_name_regexps to list
+        from rucio.core.vo import list_vos
+        vos = list_vos()
+        for vo in vos:
+            if vo['vo'] not in schema_modules:
+                load_schema_for_vo(vo['vo'])
+            scope_name_regexp = schema_modules[vo['vo']].SCOPE_NAME_REGEXP
+            if scope_name_regexp not in scope_name_regexps:
+                scope_name_regexps.append(scope_name_regexp)
+    return scope_name_regexps
+
+
+def insert_scope_name(urls: tuple[str, ...]) -> tuple[str, str]:
+    """
+    given a tuple of URLs for webpy with '%s' as a placeholder for
+    SCOPE_NAME_REGEXP, return a finalised tuple of URLs that will work for all
+    SCOPE_NAME_REGEXPs in all schemas
+    """
+
+    regexps = get_scope_name_regexps()
+    result = []
+    for i in range(0, len(urls), 2):
+        if "%s" in urls[i]:
+            # add a copy for each unique SCOPE_NAME_REGEXP
+            for scope_name_regexp in regexps:
+                result.append(urls[i] % scope_name_regexp)
+                result.append(urls[i + 1])
+        else:
+            # pass through unmodified
+            result.append(urls[i])
+            result.append(urls[i + 1])
+    return tuple(result)