rucio-clients 32.8.6__py3-none-any.whl

rucio/common/policy.py

@@ -0,0 +1,84 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import os
+from configparser import NoOptionError, NoSectionError
+from functools import wraps
+
+from dogpile.cache import make_region
+from dogpile.cache.api import NoValue
+
+from rucio.common.config import config_get
+from rucio.common.exception import UndefinedPolicy
+
+REGION = make_region().configure('dogpile.cache.memory',
+                                 expiration_time=900)
+
+
+def get_policy():
+    policy = REGION.get('policy')
+    if isinstance(policy, NoValue):
+        try:
+            policy = config_get('policy', 'permission')
+        except (NoOptionError, NoSectionError):
+            policy = 'atlas'
+        REGION.set('policy', policy)
+    return policy
+
+
+def get_scratchdisk_lifetime():
+    scratchdisk_lifetime = REGION.get('scratchdisk_lifetime')
+    if isinstance(scratchdisk_lifetime, NoValue):
+        try:
+            scratchdisk_lifetime = config_get('policy', 'scratchdisk_lifetime')
+            scratchdisk_lifetime = int(scratchdisk_lifetime)
+        except (NoOptionError, NoSectionError, ValueError):
+            scratchdisk_lifetime = 14
+        REGION.set('scratchdisk_lifetime', scratchdisk_lifetime)
+    return scratchdisk_lifetime
+
+
+def get_lifetime_policy():
+    lifetime_dict = REGION.get('lifetime_dict')
+    if isinstance(lifetime_dict, NoValue):
+        lifetime_dict = {'data': [], 'mc': [], 'valid': [], 'other': []}
+        lifetime_dir = '/opt/rucio/etc/policies'
+        try:
+            lifetime_dir = config_get('lifetime', 'directory')
+        except (NoSectionError, NoOptionError):
+            pass
+        for dtype in ['data', 'mc', 'valid', 'other']:
+            input_file_name = '%s/config_%s.json' % (lifetime_dir, dtype)
+            if os.path.isfile(input_file_name):
+                with open(input_file_name, 'r') as input_file:
+                    lifetime_dict[dtype] = json.load(input_file)
+        REGION.set('lifetime_dict', lifetime_dict)
+    return lifetime_dict
+
+
+def policy_filter(function):
+    mapping = {'atlas': ['get_scratch_policy', 'archive_localgroupdisk_datasets']}
+    policy = get_policy()
+    if policy in mapping and function.__name__ in mapping[policy]:
+        @wraps(function)
+        def new_funct(*args, **kwargs):
+            return function(*args, **kwargs)
+        return new_funct
+
+    @wraps(function)
+    def raise_funct(*args, **kwargs):
+        raise UndefinedPolicy
+    return raise_funct

rucio/common/schema/__init__.py

@@ -0,0 +1,143 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import importlib
+from configparser import NoOptionError, NoSectionError
+from os import environ
+
+from rucio.common import config, exception
+from rucio.common.utils import check_policy_package_version
+
+# dictionary of schema modules for each VO
+schema_modules = {}
+
+# list of unique SCOPE_NAME_REGEXP values from all schemas
+scope_name_regexps = []
+
+try:
+    multivo = config.config_get_bool('common', 'multi_vo', check_config_table=False)
+except (NoOptionError, NoSectionError):
+    multivo = False
+
+# multi-VO version loads schema per-VO on demand
+# we can't get a list of VOs here because the database might not
+# be available as this is imported during the bootstrapping process
+if not multivo:
+    GENERIC_FALLBACK = 'generic'
+
+    if config.config_has_section('policy'):
+        try:
+            if 'RUCIO_POLICY_PACKAGE' in environ:
+                POLICY = environ['RUCIO_POLICY_PACKAGE']
+            else:
+                POLICY = config.config_get('policy', 'package', check_config_table=False)
+            check_policy_package_version(POLICY)
+            POLICY = POLICY + ".schema"
+        except (NoOptionError, NoSectionError):
+            # fall back to old system for now
+            try:
+                POLICY = config.config_get('policy', 'schema', check_config_table=False)
+            except (NoOptionError, NoSectionError):
+                POLICY = GENERIC_FALLBACK
+            POLICY = 'rucio.common.schema.' + POLICY.lower()
+    else:
+        POLICY = 'rucio.common.schema.' + GENERIC_FALLBACK.lower()
+
+    try:
+        module = importlib.import_module(POLICY)
+    except ImportError:
+        raise exception.PolicyPackageNotFound('Module ' + POLICY + ' not found')
+
+    schema_modules["def"] = module
+    scope_name_regexps.append(module.SCOPE_NAME_REGEXP)
+
+
+def load_schema_for_vo(vo):
+    GENERIC_FALLBACK = 'generic_multi_vo'
+    if config.config_has_section('policy'):
+        try:
+            env_name = 'RUCIO_POLICY_PACKAGE_' + vo.upper()
+            if env_name in environ:
+                POLICY = environ[env_name]
+            else:
+                POLICY = config.config_get('policy', 'package-' + vo, check_config_table=False)
+            check_policy_package_version(POLICY)
+            POLICY = POLICY + ".schema"
+        except (NoOptionError, NoSectionError):
+            # fall back to old system for now
+            try:
+                POLICY = config.config_get('policy', 'schema', check_config_table=False)
+            except (NoOptionError, NoSectionError):
+                POLICY = GENERIC_FALLBACK
+            POLICY = 'rucio.common.schema.' + POLICY.lower()
+    else:
+        POLICY = 'rucio.common.schema.' + GENERIC_FALLBACK.lower()
+
+    try:
+        module = importlib.import_module(POLICY)
+    except ImportError:
+        raise exception.PolicyPackageNotFound('Module ' + POLICY + ' not found')
+
+    schema_modules[vo] = module
+
+
+def validate_schema(name, obj, vo='def'):
+    if vo not in schema_modules:
+        load_schema_for_vo(vo)
+    schema_modules[vo].validate_schema(name, obj)
+
+
+def get_schema_value(key, vo='def'):
+    if vo not in schema_modules:
+        load_schema_for_vo(vo)
+    return getattr(schema_modules[vo], key)
+
+
+def get_scope_name_regexps():
+    """ returns a list of all unique SCOPE_NAME_REGEXPs from all schemas """
+
+    if len(scope_name_regexps) == 0:
+        # load schemas for all VOs here and add unique scope_name_regexps to list
+        from rucio.core.vo import list_vos
+        vos = list_vos()
+        for vo in vos:
+            if not vo['vo'] in schema_modules:
+                load_schema_for_vo(vo['vo'])
+            scope_name_regexp = schema_modules[vo['vo']].SCOPE_NAME_REGEXP
+            if scope_name_regexp not in scope_name_regexps:
+                scope_name_regexps.append(scope_name_regexp)
+    return scope_name_regexps
+
+
+def insert_scope_name(urls):
+    """
+    given a tuple of URLs for webpy with '%s' as a placeholder for
+    SCOPE_NAME_REGEXP, return a finalised tuple of URLs that will work for all
+    SCOPE_NAME_REGEXPs in all schemas
+    """
+
+    regexps = get_scope_name_regexps()
+    result = []
+    for i in range(0, len(urls), 2):
+        if "%s" in urls[i]:
+            # add a copy for each unique SCOPE_NAME_REGEXP
+            for scope_name_regexp in regexps:
+                result.append(urls[i] % scope_name_regexp)
+                result.append(urls[i + 1])
+        else:
+            # pass through unmodified
+            result.append(urls[i])
+            result.append(urls[i + 1])
+    return tuple(result)

rucio/common/schema/atlas.py

@@ -0,0 +1,411 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from jsonschema import validate, ValidationError
+
+from rucio.common.exception import InvalidObject
+
+ACCOUNT_LENGTH = 25
+
+ACCOUNT = {"description": "Account name",
+           "type": "string",
+           "pattern": "^[a-z0-9-_]{1,%s}$" % ACCOUNT_LENGTH}
+
+ACCOUNTS = {"description": "Array of accounts",
+            "type": "array",
+            "items": ACCOUNT,
+            "minItems": 0,
+            "maxItems": 1000}
+
+
+ACCOUNT_TYPE = {"description": "Account type",
+                "type": "string",
+                "enum": ["USER", "GROUP", "SERVICE"]}
+
+ACTIVITY = {"description": "Activity name",
+            "type": "string",
+            "enum": ["Data Brokering", "Data Consolidation", "Data Rebalancing",
+                     "Debug", "Express", "Functional Test", "Functional Test XrootD",
+                     "Functional Test WebDAV", "Group Subscriptions",
+                     "Production Input", "Production Output",
+                     "Analysis Input", "Analysis Output", "Staging",
+                     "T0 Export", "T0 Tape", "Upload/Download (Job)",
+                     "Upload/Download (User)", "User Subscriptions",
+                     "Globus Online Test", "Data Challenge"]}
+
+SCOPE_LENGTH = 25
+
+SCOPE = {"description": "Scope name",
+         "type": "string",
+         "pattern": "^[a-zA-Z_\\-.0-9]{1,%s}$" % SCOPE_LENGTH}
+
+R_SCOPE = {"description": "Scope name",
+           "type": "string",
+           "pattern": "\\w"}
+
+NAME_LENGTH = 250
+
+NAME = {"description": "Data Identifier name",
+        "type": "string",
+        "pattern": "^[A-Za-z0-9][A-Za-z0-9\\.\\-\\_]{1,%s}$" % NAME_LENGTH}
+
+R_NAME = {"description": "Data Identifier name",
+          "type": "string",
+          "pattern": "\\w"}
+
+LOCKED = {"description": "Rule locked status",
+          "type": ["boolean", "null"]}
+
+ASK_APPROVAL = {"description": "Rule approval request",
+                "type": ["boolean", "null"]}
+
+ASYNCHRONOUS = {"description": "Asynchronous rule creation",
+                "type": ["boolean", "null"]}
+
+DELAY_INJECTION = {"description": "Time (in seconds) to wait before starting applying the rule. Implies asynchronous rule creation.",
+                   "type": ["integer", "null"]}
+
+PURGE_REPLICAS = {"description": "Rule purge replica status",
+                  "type": "boolean"}
+
+IGNORE_AVAILABILITY = {"description": "Rule ignore availability status",
+                       "type": "boolean"}
+
+RSE = {"description": "RSE name",
+       "type": "string",
+       "pattern": "^([A-Z0-9]+([_-][A-Z0-9]+)*)$"}
+
+RSE_ATTRIBUTE = {"description": "RSE attribute",
+                 "type": "string",
+                 "pattern": r'([A-Za-z0-9\._-]+[=<>][A-Za-z0-9_-]+)'}
+
+DEFAULT_RSE_ATTRIBUTE = {"description": "Default RSE attribute",
+                         "type": "string",
+                         "pattern": r'([A-Z0-9]+([_-][A-Z0-9]+)*)'}
+
+REPLICA_STATE = {"description": "Replica state",
+                 "type": "string",
+                 "enum": ["AVAILABLE", "UNAVAILABLE", "COPYING", "BEING_DELETED", "BAD", "SOURCE", "A", "U", "C", "B", "D", "S"]}
+
+DATE = {"description": "Date",
+        "type": "string",
+        "pattern": r'((Mon)|(Tue)|(Wed)|(Thu)|(Fri)|(Sat)|(Sun))[,]\s\d{2}\s(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s\d{4}\s(0\d|1\d|2[0-3])(\:)(0\d|1\d|2\d|3\d|4\d|5\d)(\:)(0\d|1\d|2\d|3\d|4\d|5\d)\s(UTC)'}
+
+DID_TYPE = {"description": "DID type",
+            "type": "string",
+            "enum": ["DATASET", "CONTAINER", "FILE", "F"]}
+
+GROUPING = {"description": "Rule grouping",
+            "type": ["string", "null"],
+            "enum": ["DATASET", "NONE", "ALL", None]}
+
+NOTIFY = {"description": "Rule notification setting",
+          "type": ["string", "null"],
+          "enum": ["Y", "C", "N", "P", None]}
+
+COMMENT = {"description": "Rule comment",
+           "type": ["string", "null"],
+           "maxLength": 250}
+
+METADATA = {"description": "Rule wfms metadata",
+            "type": ["string", "null"],
+            "maxLength": 3999}
+
+BYTES = {"description": "Size in bytes",
+         "type": "integer"}
+
+ADLER32 = {"description": "adler32",
+           "type": "string",
+           "pattern": "^[a-fA-F\\d]{8}$"}
+
+WEIGHT = {"description": "Rule weight",
+          "type": ["string", "null"]}
+
+MD5 = {"description": "md5",
+       "type": "string",
+       "pattern": "^[a-fA-F\\d]{32}$"}
+
+UUID = {"description": "Universally Unique Identifier (UUID)",
+        "type": "string",
+        "pattern": '^(\\{){0,1}[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}(\\}){0,1}$'}
+
+META = {"description": "Data Identifier(DID) metadata",
+        "type": "object",
+        "properties": {"guid": UUID},
+        "additionalProperties": True}
+
+PFN = {"description": "Physical File Name", "type": "string"}
+
+COPIES = {"description": "Number of replica copies", "type": "integer"}
+
+RSE_EXPRESSION = {"description": "RSE expression", "type": "string"}
+
+SOURCE_REPLICA_EXPRESSION = {"description": "RSE expression", "type": ["string", "null"]}
+
+LIFETIME = {"description": "Lifetime", "type": "number"}
+
+RULE_LIFETIME = {"description": "Rule lifetime", "type": ["number", "null"]}
+
+SUBSCRIPTION_ID = {"description": "Rule Subscription id", "type": ["string", "null"]}
+
+PRIORITY = {"description": "Priority of the transfers",
+            "type": "integer"}
+
+SPLIT_CONTAINER = {"description": "Rule split container mode",
+                   "type": ["boolean", "null"]}
+
+RULE = {"description": "Replication rule",
+        "type": "object",
+        "properties": {"dids": {"type": "array"},
+                       "account": ACCOUNT,
+                       "copies": COPIES,
+                       "rse_expression": RSE_EXPRESSION,
+                       "grouping": GROUPING,
+                       "weight": WEIGHT,
+                       "lifetime": RULE_LIFETIME,
+                       "locked": LOCKED,
+                       "subscription_id": SUBSCRIPTION_ID,
+                       "source_replica_expression": SOURCE_REPLICA_EXPRESSION,
+                       "activity": ACTIVITY,
+                       "notify": NOTIFY,
+                       "purge_replicas": PURGE_REPLICAS,
+                       "ignore_availability": IGNORE_AVAILABILITY,
+                       "comment": COMMENT,
+                       "ask_approval": ASK_APPROVAL,
+                       "asynchronous": ASYNCHRONOUS,
+                       "delay_injection": DELAY_INJECTION,
+                       "priority": PRIORITY,
+                       'split_container': SPLIT_CONTAINER,
+                       'meta': METADATA},
+        "required": ["dids", "copies", "rse_expression"],
+        "additionalProperties": False}
+
+RULES = {"description": "Array of replication rules",
+         "type": "array",
+         "items": RULE,
+         "minItems": 1,
+         "maxItems": 1000}
+
+COLLECTION_TYPE = {"description": "Dataset or container type",
+                   "type": "string",
+                   "enum": ["DATASET", "CONTAINER"]}
+
+COLLECTION = {"description": "Dataset or container",
+              "type": "object",
+              "properties": {"scope": SCOPE,
+                             "name": NAME,
+                             "type": COLLECTION_TYPE,
+                             "meta": META,
+                             "rules": RULES},
+              "required": ["scope", "name", "type"],
+              "additionalProperties": False}
+
+COLLECTIONS = {"description": "Array of datasets or containers",
+               "type": "array",
+               "items": COLLECTION,
+               "minItems": 1,
+               "maxItems": 1000}
+
+DID = {"description": "Data Identifier(DID)",
+       "type": "object",
+       "properties": {"scope": SCOPE,
+                      "name": NAME,
+                      "type": DID_TYPE,
+                      "meta": META,
+                      "rules": RULES,
+                      "bytes": BYTES,
+                      "adler32": ADLER32,
+                      "md5": MD5,
+                      "state": REPLICA_STATE,
+                      "pfn": PFN},
+       "required": ["scope", "name"],
+       "additionalProperties": False}
+
+DID_FILTERS = {"description": "Array to filter DIDs by metadata",
+               "type": "array",
+               "additionalProperties": True}
+
+R_DID = {"description": "Data Identifier(DID)",
+         "type": "object",
+         "properties": {"scope": R_SCOPE,
+                        "name": R_NAME,
+                        "type": DID_TYPE,
+                        "meta": META,
+                        "rules": RULES,
+                        "bytes": BYTES,
+                        "adler32": ADLER32,
+                        "md5": MD5,
+                        "state": REPLICA_STATE,
+                        "pfn": PFN},
+         "required": ["scope", "name"],
+         "additionalProperties": False}
+
+DIDS = {"description": "Array of Data Identifiers(DIDs)",
+        "type": "array",
+        "items": DID,
+        "minItems": 1,
+        "maxItems": 1000}
+
+R_DIDS = {"description": "Array of Data Identifiers(DIDs)",
+          "type": "array",
+          "items": R_DID,
+          "minItems": 1,
+          "maxItems": 1000}
+
+ATTACHMENT = {"description": "Attachement",
+              "type": "object",
+              "properties": {"scope": SCOPE,
+                             "name": NAME,
+                             "rse": {"description": "RSE name",
+                                     "type": ["string", "null"],
+                                     "pattern": "^([A-Z0-9]+([_-][A-Z0-9]+)*)$"},
+                             "dids": DIDS},
+              "required": ["dids"],
+              "additionalProperties": False}
+
+ATTACHMENTS = {"description": "Array of attachments",
+               "type": "array",
+               "items": ATTACHMENT,
+               "minItems": 1,
+               "maxItems": 1000}
+
+SUBSCRIPTION_FILTER = {"type": "object",
+                       "properties": {"datatype": {"type": "array"},
+                                      "prod_step": {"type": "array"},
+                                      "stream_name": {"type": "array"},
+                                      "project": {"type": "array"},
+                                      "scope": {"type": "array"},
+                                      "pattern": {"type": "string"},
+                                      "excluded_pattern": {"type": "string"},
+                                      "group": {"type": "string"},
+                                      "provenance": {"type": "string"},
+                                      "account": ACCOUNTS,
+                                      "grouping": {"type": "string"},
+                                      "split_rule": {"type": "boolean"}}}
+
+ADD_REPLICA_FILE = {"description": "add replica file",
+                    "type": "object",
+                    "properties": {"scope": SCOPE,
+                                   "name": NAME,
+                                   "bytes": BYTES,
+                                   "adler32": ADLER32},
+                    "required": ["scope", "name", "bytes", "adler32"]}
+
+ADD_REPLICA_FILES = {"description": "add replica files",
+                     "type": "array",
+                     "items": ADD_REPLICA_FILE,
+                     "minItems": 1,
+                     "maxItems": 1000}
+
+CACHE_ADD_REPLICAS = {"description": "rucio cache add replicas",
+                      "type": "object",
+                      "properties": {"files": ADD_REPLICA_FILES,
+                                     "rse": RSE,
+                                     "lifetime": LIFETIME,
+                                     "operation": {"enum": ["add_replicas"]}},
+                      "required": ['files', 'rse', 'lifetime', 'operation']}
+
+DELETE_REPLICA_FILE = {"description": "delete replica file",
+                       "type": "object",
+                       "properties": {"scope": SCOPE,
+                                      "name": NAME},
+                       "required": ["scope", "name"]}
+
+DELETE_REPLICA_FILES = {"description": "delete replica files",
+                        "type": "array",
+                        "items": DELETE_REPLICA_FILE,
+                        "minItems": 1,
+                        "maxItems": 1000}
+
+CACHE_DELETE_REPLICAS = {"description": "rucio cache delete replicas",
+                         "type": "object",
+                         "properties": {"files": DELETE_REPLICA_FILES,
+                                        "rse": RSE,
+                                        "operation": {"enum": ["delete_replicas"]}},
+                         "required": ['files', 'rse', 'operation']}
+
+MESSAGE_OPERATION = {"type": "object",
+                     "properties": {'operation': {"enum": ["add_replicas", "delete_replicas"]}}}
+
+ACCOUNT_ATTRIBUTE = {"description": "Account attribute",
+                     "type": "string",
+                     "pattern": r'^[a-zA-Z0-9-_\\/\\.]{1,30}$'}
+
+SCOPE_NAME_REGEXP = '/(.*)/(.*)'
+
+DISTANCE = {"description": "RSE distance",
+            "type": "object",
+            "properties": {
+                "src_rse_id": {"type": "string"},
+                "dest_rse_id": {"type": "string"},
+                "ranking": {"type": "integer"}
+            },
+            "required": ["src_rse_id", "dest_rse_id", "ranking"],
+            "additionalProperties": True}
+
+IMPORT = {"description": "import data into rucio.",
+          "type": "object",
+          "properties": {
+              "rses": {
+                  "type": "object"
+              },
+              "distances": {
+                  "type": "object"
+              }
+          }}
+
+VO = {"description": "VO tag",
+      "type": "string",
+      "pattern": "^([a-zA-Z_\\-.0-9]{3})?$"}
+
+SCHEMAS = {'account': ACCOUNT,
+           'account_type': ACCOUNT_TYPE,
+           'activity': ACTIVITY,
+           'name': NAME,
+           'r_name': R_NAME,
+           'rse': RSE,
+           'rse_attribute': RSE_ATTRIBUTE,
+           'scope': SCOPE,
+           'r_scope': R_SCOPE,
+           'did': DID,
+           'did_filters': DID_FILTERS,
+           'r_did': R_DID,
+           'dids': DIDS,
+           'rule': RULE,
+           'r_dids': R_DIDS,
+           'collection': COLLECTION,
+           'collections': COLLECTIONS,
+           'attachment': ATTACHMENT,
+           'attachments': ATTACHMENTS,
+           'subscription_filter': SUBSCRIPTION_FILTER,
+           'cache_add_replicas': CACHE_ADD_REPLICAS,
+           'cache_delete_replicas': CACHE_DELETE_REPLICAS,
+           'account_attribute': ACCOUNT_ATTRIBUTE,
+           'import': IMPORT,
+           'vo': VO}
+
+
+def validate_schema(name, obj):
+    """
+    Validate object against json schema
+
+    :param name: The json schema name.
+    :param obj: The object to validate.
+    """
+    try:
+        if obj:
+            validate(obj, SCHEMAS.get(name, {}))
+    except ValidationError as error:  # NOQA, pylint: disable=W0612
+        raise InvalidObject(f'Problem validating {name}: {error}')