rucio-clients 35.7.0__py3-none-any.whl

rucio/common/plugins.py

@@ -0,0 +1,153 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import importlib
+import os
+from collections.abc import Callable
+from configparser import NoOptionError, NoSectionError
+from typing import Any, TypeVar
+
+from rucio.common import config
+from rucio.common.exception import InvalidAlgorithmName
+
+PolicyPackageAlgorithmsT = TypeVar('PolicyPackageAlgorithmsT', bound='PolicyPackageAlgorithms')
+
+
+class PolicyPackageAlgorithms:
+    """
+    Base class for Rucio Policy Package Algorithms
+
+    ALGORITHMS is of type Dict[str, Dict[str. Callable[..., Any]]]
+    where the key is the algorithm type and the value is a dictionary of algorithm names and their callables
+    """
+    _ALGORITHMS: dict[str, dict[str, Callable[..., Any]]] = {}
+    _loaded_policy_modules = False
+
+    def __init__(self) -> None:
+        if not self._loaded_policy_modules:
+            self._register_all_policy_package_algorithms()
+            self._loaded_policy_modules = True
+
+    @classmethod
+    def _get_one_algorithm(cls: type[PolicyPackageAlgorithmsT], algorithm_type: str, name: str) -> Callable[..., Any]:
+        """
+        Get the algorithm from the dictionary of algorithms
+        """
+        return cls._ALGORITHMS[algorithm_type][name]
+
+    @classmethod
+    def _get_algorithms(cls: type[PolicyPackageAlgorithmsT], algorithm_type: str) -> dict[str, Callable[..., Any]]:
+        """
+        Get the dictionary of algorithms for a given type
+        """
+        return cls._ALGORITHMS[algorithm_type]
+
+    @classmethod
+    def _register(
+            cls: type[PolicyPackageAlgorithmsT],
+            algorithm_type: str, algorithm_dict: dict[str, Callable[..., Any]]) -> None:
+        """
+        Provided a dictionary of callable function,
+        and the associated algorithm type,
+        register it as one of the valid algorithms.
+        """
+        if algorithm_type in cls._ALGORITHMS:
+            cls._ALGORITHMS[algorithm_type].update(algorithm_dict)
+        else:
+            cls._ALGORITHMS[algorithm_type] = algorithm_dict
+
+    @classmethod
+    def _supports(cls: type[PolicyPackageAlgorithmsT], algorithm_type: str, name: str) -> bool:
+        """
+        Check if a algorithm is supported by the plugin
+        """
+        return name in cls._ALGORITHMS.get(algorithm_type, {})
+
+    @classmethod
+    def _register_all_policy_package_algorithms(cls: type[PolicyPackageAlgorithmsT]) -> None:
+        '''
+        Loads all the algorithms of a given type from the policy package(s) and registers them
+        :param algorithm_type: the type of algorithm to register (e.g. 'lfn2pfn')
+        :param dictionary: the dictionary to register them in
+        :param vo: the name of the relevant VO (None for single VO)
+        '''
+        try:
+            multivo = config.config_get_bool('common', 'multi_vo')
+        except (NoOptionError, NoSectionError):
+            multivo = False
+        if not multivo:
+            # single policy package
+            cls._try_importing_policy()
+        else:
+            # determine whether on client or server
+            client = False
+            if 'RUCIO_CLIENT_MODE' not in os.environ:
+                if not config.config_has_section('database') and config.config_has_section('client'):
+                    client = True
+            else:
+                if os.environ['RUCIO_CLIENT_MODE']:
+                    client = True
+
+            # on client, only register algorithms for selected VO
+            if client:
+                if 'RUCIO_VO' in os.environ:
+                    vo = os.environ['RUCIO_VO']
+                else:
+                    try:
+                        vo = str(config.config_get('client', 'vo'))
+                    except (NoOptionError, NoSectionError):
+                        vo = 'def'
+                cls._try_importing_policy(vo)
+            # on server, list all VOs and register their algorithms
+            else:
+                from rucio.core.vo import list_vos
+                # policy package per VO
+                vos = list_vos()
+                for vo in vos:
+                    cls._try_importing_policy(vo['vo'])
+
+    @classmethod
+    def _try_importing_policy(cls: type[PolicyPackageAlgorithmsT], vo: str = "") -> None:
+        try:
+            # import from utils here to avoid circular import
+            from rucio.common.utils import check_policy_package_version
+
+            env_name = 'RUCIO_POLICY_PACKAGE' + ('' if not vo else '_' + vo.upper())
+            package = getattr(os.environ, env_name, "")
+            if not package:
+                package = str(config.config_get('policy', 'package' + ('' if not vo else '-' + vo)))
+
+            check_policy_package_version(package)
+            module = importlib.import_module(package)
+
+            if hasattr(module, 'get_algorithms'):
+                all_algorithms = module.get_algorithms()
+
+                # for backward compatibility, rename 'surl' to 'non_deterministic_pfn' here
+                if 'surl' in all_algorithms:
+                    all_algorithms['non_deterministic_pfn'] = all_algorithms['surl']
+
+                # check that the names are correctly prefixed for multi-VO
+                if vo:
+                    for _, algorithms in all_algorithms.items():
+                        for k in algorithms.keys():
+                            if not k.lower().startswith(vo.lower()):
+                                raise InvalidAlgorithmName(k, vo)
+
+                # Updates the dictionary with the algorithms from the policy package
+                for algorithm_type, algorithm_dict in all_algorithms.items():
+                    cls._register(algorithm_type, algorithm_dict)
+
+        except (NoOptionError, NoSectionError, ImportError):
+            pass

rucio/common/policy.py

@@ -0,0 +1,84 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import os
+from configparser import NoOptionError, NoSectionError
+from functools import wraps
+from typing import Any
+
+from dogpile.cache import make_region
+from dogpile.cache.api import NoValue
+
+from rucio.common.config import config_get
+from rucio.common.exception import UndefinedPolicy
+
+REGION = make_region().configure('dogpile.cache.memory',
+                                 expiration_time=900)
+
+
+def get_policy() -> str:
+    policy = REGION.get('policy')
+    if isinstance(policy, NoValue):
+        try:
+            policy = config_get('policy', 'permission')
+        except (NoOptionError, NoSectionError):
+            policy = 'atlas'
+        REGION.set('policy', policy)
+    return policy
+
+
+def get_scratchdisk_lifetime() -> int:
+    scratchdisk_lifetime = REGION.get('scratchdisk_lifetime')
+    if isinstance(scratchdisk_lifetime, NoValue):
+        try:
+            scratchdisk_lifetime = config_get('policy', 'scratchdisk_lifetime')
+            scratchdisk_lifetime = int(scratchdisk_lifetime)
+        except (NoOptionError, NoSectionError, ValueError):
+            scratchdisk_lifetime = 14
+        REGION.set('scratchdisk_lifetime', scratchdisk_lifetime)
+    return scratchdisk_lifetime
+
+
+def get_lifetime_policy() -> dict[str, list[dict[str, Any]]]:
+    lifetime_dict = REGION.get('lifetime_dict')
+    if isinstance(lifetime_dict, NoValue):
+        lifetime_dict = {'data': [], 'mc': [], 'valid': [], 'other': []}
+        lifetime_dir = '/opt/rucio/etc/policies'
+        try:
+            lifetime_dir = config_get('lifetime', 'directory')
+        except (NoSectionError, NoOptionError):
+            pass
+        for dtype in ['data', 'mc', 'valid', 'other']:
+            input_file_name = '%s/config_%s.json' % (lifetime_dir, dtype)
+            if os.path.isfile(input_file_name):
+                with open(input_file_name, 'r') as input_file:
+                    lifetime_dict[dtype] = json.load(input_file)
+        REGION.set('lifetime_dict', lifetime_dict)
+    return lifetime_dict
+
+
+def policy_filter(function):
+    mapping = {'atlas': ['get_scratch_policy', 'archive_localgroupdisk_datasets']}
+    policy = get_policy()
+    if policy in mapping and function.__name__ in mapping[policy]:
+        @wraps(function)
+        def new_funct(*args, **kwargs):
+            return function(*args, **kwargs)
+        return new_funct
+
+    @wraps(function)
+    def raise_funct(*args, **kwargs):
+        raise UndefinedPolicy
+    return raise_funct

rucio/common/schema/__init__.py

@@ -0,0 +1,150 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import importlib
+from configparser import NoOptionError, NoSectionError
+from os import environ
+from typing import TYPE_CHECKING, Any
+
+from rucio.common import config, exception
+from rucio.common.utils import check_policy_package_version
+
+if TYPE_CHECKING:
+    from types import ModuleType
+
+# dictionary of schema modules for each VO
+schema_modules: dict[str, "ModuleType"] = {}
+
+# list of unique SCOPE_NAME_REGEXP values from all schemas
+scope_name_regexps: list[str] = []
+
+try:
+    multivo = config.config_get_bool('common', 'multi_vo', check_config_table=False)
+except (NoOptionError, NoSectionError):
+    multivo = False
+
+# multi-VO version loads schema per-VO on demand
+# we can't get a list of VOs here because the database might not
+# be available as this is imported during the bootstrapping process
+if not multivo:
+    GENERIC_FALLBACK = 'generic'
+
+    if config.config_has_section('policy'):
+        try:
+            if 'RUCIO_POLICY_PACKAGE' in environ:
+                POLICY = environ['RUCIO_POLICY_PACKAGE']
+            else:
+                POLICY = config.config_get('policy', 'package', check_config_table=False)
+            check_policy_package_version(POLICY)
+            POLICY = POLICY + ".schema"
+        except (NoOptionError, NoSectionError):
+            # fall back to old system for now
+            try:
+                POLICY = config.config_get('policy', 'schema', check_config_table=False)
+            except (NoOptionError, NoSectionError):
+                POLICY = GENERIC_FALLBACK
+            POLICY = 'rucio.common.schema.' + POLICY.lower()
+    else:
+        POLICY = 'rucio.common.schema.' + GENERIC_FALLBACK.lower()
+
+    try:
+        module = importlib.import_module(POLICY)
+    except ModuleNotFoundError:
+        raise exception.PolicyPackageNotFound(POLICY)
+    except ImportError:
+        raise exception.ErrorLoadingPolicyPackage(POLICY)
+
+    schema_modules["def"] = module
+    scope_name_regexps.append(module.SCOPE_NAME_REGEXP)
+
+
+def load_schema_for_vo(vo: str) -> None:
+    GENERIC_FALLBACK = 'generic_multi_vo'
+    if config.config_has_section('policy'):
+        try:
+            env_name = 'RUCIO_POLICY_PACKAGE_' + vo.upper()
+            if env_name in environ:
+                POLICY = environ[env_name]
+            else:
+                POLICY = config.config_get('policy', 'package-' + vo, check_config_table=False)
+            check_policy_package_version(POLICY)
+            POLICY = POLICY + ".schema"
+        except (NoOptionError, NoSectionError):
+            # fall back to old system for now
+            try:
+                POLICY = config.config_get('policy', 'schema', check_config_table=False)
+            except (NoOptionError, NoSectionError):
+                POLICY = GENERIC_FALLBACK
+            POLICY = 'rucio.common.schema.' + POLICY.lower()
+    else:
+        POLICY = 'rucio.common.schema.' + GENERIC_FALLBACK.lower()
+
+    try:
+        module = importlib.import_module(POLICY)
+    except ModuleNotFoundError:
+        raise exception.PolicyPackageNotFound(POLICY)
+    except ImportError:
+        raise exception.ErrorLoadingPolicyPackage(POLICY)
+
+    schema_modules[vo] = module
+
+
+def validate_schema(name: str, obj: Any, vo: str = 'def') -> None:
+    if vo not in schema_modules:
+        load_schema_for_vo(vo)
+    schema_modules[vo].validate_schema(name, obj)
+
+
+def get_schema_value(key: str, vo: str = 'def') -> Any:
+    if vo not in schema_modules:
+        load_schema_for_vo(vo)
+    return getattr(schema_modules[vo], key)
+
+
+def get_scope_name_regexps() -> list[str]:
+    """ returns a list of all unique SCOPE_NAME_REGEXPs from all schemas """
+
+    if len(scope_name_regexps) == 0:
+        # load schemas for all VOs here and add unique scope_name_regexps to list
+        from rucio.core.vo import list_vos
+        vos = list_vos()
+        for vo in vos:
+            if not vo['vo'] in schema_modules:
+                load_schema_for_vo(vo['vo'])
+            scope_name_regexp = schema_modules[vo['vo']].SCOPE_NAME_REGEXP
+            if scope_name_regexp not in scope_name_regexps:
+                scope_name_regexps.append(scope_name_regexp)
+    return scope_name_regexps
+
+
+def insert_scope_name(urls: tuple[str, ...]) -> tuple[str, str]:
+    """
+    given a tuple of URLs for webpy with '%s' as a placeholder for
+    SCOPE_NAME_REGEXP, return a finalised tuple of URLs that will work for all
+    SCOPE_NAME_REGEXPs in all schemas
+    """
+
+    regexps = get_scope_name_regexps()
+    result = []
+    for i in range(0, len(urls), 2):
+        if "%s" in urls[i]:
+            # add a copy for each unique SCOPE_NAME_REGEXP
+            for scope_name_regexp in regexps:
+                result.append(urls[i] % scope_name_regexp)
+                result.append(urls[i + 1])
+        else:
+            # pass through unmodified
+            result.append(urls[i])
+            result.append(urls[i + 1])
+    return tuple(result)