rucio-clients 37.0.0rc1__py3-none-any.whl

rucio/common/constants.py

@@ -0,0 +1,217 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import enum
+from collections import namedtuple
+from typing import Literal, get_args
+
+from rucio.common.config import config_get_bool
+
+"""
+Constants.
+
+"""
+
+RESERVED_KEYS = ['scope', 'name', 'account', 'did_type', 'is_open', 'monotonic', 'obsolete', 'complete',
+                 'availability', 'suppressed', 'bytes', 'length', 'md5', 'adler32', 'rule_evaluation_action',
+                 'rule_evaluation_required', 'expired_at', 'deleted_at', 'created_at', 'updated_at']
+# collection_keys =
+# file_keys =
+
+KEY_TYPES = ['ALL', 'COLLECTION', 'FILE', 'DERIVED']
+# all(container, dataset, file), collection(dataset or container), file, derived(compute from file for collection)
+
+SCHEME_MAP = {'srm': ['srm', 'gsiftp'],
+              'gsiftp': ['srm', 'gsiftp'],
+              'https': ['https', 'davs', 'srm+https', 'cs3s'],
+              'davs': ['https', 'davs', 'srm+https', 'cs3s'],
+              'srm+https': ['https', 'davs', 'srm+https', 'cs3s'],
+              'cs3s': ['https', 'davs', 'srm+https', 'cs3s'],
+              'root': ['root'],
+              'scp': ['scp'],
+              'rsync': ['rsync'],
+              'rclone': ['rclone']}
+if config_get_bool('transfers', 'srm_https_compatibility', raise_exception=False, default=False):
+    SCHEME_MAP['srm'].append('https')
+    SCHEME_MAP['https'].append('srm')
+    SCHEME_MAP['srm'].append('davs')
+    SCHEME_MAP['davs'].append('srm')
+
+SORTING_ALGORITHMS_LITERAL = Literal['geoip', 'custom_table', 'random']
+SORTING_ALGORITHMS = list(get_args(SORTING_ALGORITHMS_LITERAL))
+
+SUPPORTED_PROTOCOLS_LITERAL = Literal['gsiftp', 'srm', 'root', 'davs', 'http', 'https', 'file', 'storm', 'srm+https', 'scp', 'rsync', 'rclone', 'magnet']
+SUPPORTED_PROTOCOLS: list[str] = list(get_args(SUPPORTED_PROTOCOLS_LITERAL))
+
+RSE_SUPPORTED_PROTOCOL_DOMAINS_LITERAL = Literal['ALL', 'LAN', 'WAN']
+
+RSE_BASE_SUPPORTED_PROTOCOL_OPERATIONS_LITERAL = Literal['read', 'write', 'delete']
+RSE_BASE_SUPPORTED_PROTOCOL_OPERATIONS: list[str] = list(get_args(RSE_BASE_SUPPORTED_PROTOCOL_OPERATIONS_LITERAL))
+
+RSE_ALL_SUPPORTED_PROTOCOL_OPERATIONS_LITERAL = Literal[RSE_BASE_SUPPORTED_PROTOCOL_OPERATIONS_LITERAL, 'third_party_copy_read', 'third_party_copy_write']
+RSE_ALL_SUPPORTED_PROTOCOL_OPERATIONS: list[str] = list(get_args(RSE_ALL_SUPPORTED_PROTOCOL_OPERATIONS_LITERAL))
+
+FTS_STATE = namedtuple('FTS_STATE', ['SUBMITTED', 'READY', 'ACTIVE', 'FAILED', 'FINISHED', 'FINISHEDDIRTY', 'NOT_USED',
+                                     'CANCELED'])('SUBMITTED', 'READY', 'ACTIVE', 'FAILED', 'FINISHED', 'FINISHEDDIRTY',
+                                                  'NOT_USED', 'CANCELED')
+
+FTS_COMPLETE_STATE = namedtuple('FTS_COMPLETE_STATE', ['OK', 'ERROR'])('Ok', 'Error')
+
+# https://gitlab.cern.ch/fts/fts3/-/blob/master/src/db/generic/Job.h#L41
+FTS_JOB_TYPE = namedtuple('FTS_JOB_TYPE', ['MULTIPLE_REPLICA', 'MULTI_HOP', 'SESSION_REUSE', 'REGULAR'])('R', 'H', 'Y', 'N')
+
+
+# Messages constants
+
+MAX_MESSAGE_LENGTH = 4000
+
+
+@enum.unique
+class SuspiciousAvailability(enum.Enum):
+    ALL = 0
+    EXIST_COPIES = 1
+    LAST_COPY = 2
+
+
+@enum.unique
+class ReplicaState(enum.Enum):
+    # From rucio.db.sqla.constants, update that file at the same time as this
+    AVAILABLE = 'A'
+    UNAVAILABLE = 'U'
+    COPYING = 'C'
+    BEING_DELETED = 'B'
+    BAD = 'D'
+    TEMPORARY_UNAVAILABLE = 'T'
+
+
+@enum.unique
+class HermesService(str, enum.Enum):
+    """
+    The services supported by Hermes2.
+    """
+    INFLUX = "INFLUX"
+    ELASTIC = "ELASTIC"
+    EMAIL = "EMAIL"
+    ACTIVEMQ = "ACTIVEMQ"
+
+
+class RseAttr:
+
+    """
+    List of functional RSE attributes.
+
+    This class acts as a namespace containing all RSE attributes referenced in
+    the Rucio source code. Setting them affects Rucio's behaviour in some way.
+    """
+
+    ARCHIVE_TIMEOUT = 'archive_timeout'
+    ASSOCIATED_SITES = 'associated_sites'
+    AUTO_APPROVE_BYTES = 'auto_approve_bytes'
+    AUTO_APPROVE_FILES = 'auto_approve_files'
+    BITTORRENT_TRACKER_ADDR = 'bittorrent_tracker_addr'
+    BLOCK_MANUAL_APPROVAL = 'block_manual_approval'
+    COUNTRY = 'country'
+    DECOMMISSION = 'decommission'
+    DEFAULT_ACCOUNT_LIMIT_BYTES = 'default_account_limit_bytes'
+    FTS = 'fts'
+    GLOBUS_ENDPOINT_ID = 'globus_endpoint_id'
+    GREEDYDELETION = 'greedyDeletion'
+    IS_OBJECT_STORE = 'is_object_store'
+    LFN2PFN_ALGORITHM = 'lfn2pfn_algorithm'
+    MAXIMUM_PIN_LIFETIME = 'maximum_pin_lifetime'
+    MULTIHOP_TOMBSTONE_DELAY = 'multihop_tombstone_delay'
+    NAMING_CONVENTION = 'naming_convention'
+    OIDC_BASE_PATH = 'oidc_base_path'
+    OIDC_SUPPORT = 'oidc_support'
+    PHYSGROUP = 'physgroup'
+    QBITTORRENT_MANAGEMENT_ADDRESS = 'qbittorrent_management_address'
+    RESTRICTED_READ = 'restricted_read'
+    RESTRICTED_WRITE = 'restricted_write'
+    RULE_APPROVERS = 'rule_approvers'
+    S3_URL_STYLE = 's3_url_style'
+    SIGN_URL = 'sign_url'
+    SIMULATE_MULTIRANGE = 'simulate_multirange'
+    SITE = 'site'
+    SKIP_UPLOAD_STAT = 'skip_upload_stat'
+    SOURCE_FOR_TOTAL_SPACE = 'source_for_total_space'
+    SOURCE_FOR_USED_SPACE = 'source_for_used_space'
+    STAGING_BUFFER = 'staging_buffer'
+    STAGING_REQUIRED = 'staging_required'
+    STRICT_COPY = 'strict_copy'
+    TOMBSTONE_DELAY = 'tombstone_delay'
+    TYPE = 'type'
+    USE_IPV4 = 'use_ipv4'
+    VERIFY_CHECKSUM = 'verify_checksum'
+
+    # The following RSE attributes are exclusively used in the permission layer
+    # and are likely VO-specific.
+
+    BLOCK_MANUAL_APPROVE = 'block_manual_approve'
+    CMS_TYPE = 'cms_type'
+    DEFAULT_LIMIT_FILES = 'default_limit_files'
+    QUOTA_APPROVERS = 'quota_approvers'
+    RULE_DELETERS = 'rule_deleters'
+
+
+# Literal types to allow overloading of functions with RSE attributes in their signature.
+# RSE attributes are encoded via the BooleanString decorator as VARCHAR in the database,
+# but they are used as either bool or string in the code.
+# This is only determined at runtime, so for static type checking
+# we need to manually specify which attrs are string and which are bool.
+# In future, we could refactor RseAttr to avoid code duplication.
+RSE_ATTRS_STR = Literal[
+    'archive_timeout',
+    'associated_sites',
+    'bittorrent_tracker_addr',
+    'country',
+    'decommission',
+    'default_account_limit_bytes',
+    'fts',
+    'globus_endpoint_id',
+    'lfn2pfn_algorithm',
+    'maximum_pin_lifetime',
+    'multihop_tombstone_delay',
+    'naming_convention',
+    'oidc_base_path',
+    'oidc_support'
+    'physgroup',
+    'qbittorrent_management_address'
+    'rule_approvers',
+    's3_url_style',
+    'simulate_multirange',
+    'site',
+    'source_for_total_space',
+    'source_for_used_space',
+    'staging_buffer',
+    'tombstone_delay',
+    'type'
+]
+
+RSE_ATTRS_BOOL = Literal[
+    'auto_approve_bytes',
+    'auto_approve_files',
+    'block_manual_approval',
+    'greedyDeletion',
+    'is_object_store',
+    'restricted_read',
+    'restricted_write',
+    'skip_upload_stat',
+    'staging_required',
+    'strict_copy',
+    'use_ipv4',
+    'verify_checksum'
+]
+
+SUPPORTED_SIGN_URL_SERVICES_LITERAL = Literal['gcs', 's3', 'swift']
+SUPPORTED_SIGN_URL_SERVICES = list(get_args(SUPPORTED_SIGN_URL_SERVICES_LITERAL))

rucio/common/constraints.py

@@ -0,0 +1,17 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# List of authorized value types for key
+AUTHORIZED_VALUE_TYPES = (float, int, str)
+STRING_TYPES = (str, )

rucio/common/didtype.py

@@ -0,0 +1,237 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+DID type to represent a did and to simplify operations on it
+"""
+
+from typing import Any, Union
+
+from rucio.common.exception import DIDError
+
+
+class DID:
+
+    """
+    Class used to store a DID
+    Given an object did of type DID
+    scope is stored in did.scope
+    name is stored in did.name
+    """
+
+    SCOPE_SEPARATOR = ':'
+    IMPLICIT_SCOPE_SEPARATOR = '.'
+    IMPLICIT_SCOPE_TO_LEN = {'user': 2, 'group': 2}
+    __slots__ = ['scope', 'name']
+
+    def __init__(self, *args, **kwargs):
+        """
+        Constructs the DID object. Possible parameter combinations are:
+            DID()
+            DID('scope:name.did.str')
+            DID('user.implicit.scope.in.name')
+            DID('custom.scope', 'custom.name')
+            DID(['list.scope', 'list.name'])
+            DID(('tuple.scope', 'tuple.name'))
+            DID({'scope': 'dict.scope', 'name': 'dict.name'})
+            DID(scope='kw.scope')
+            DID(name='kw.name')
+            DID(name='user.kw.implicit.scope')
+            DID(scope='kw.scope', name='kw.name')
+            DID(did={'scope': 'kw.did.scope', 'name': 'kw.did.name'})
+            DID(did=['kw.list.scope', 'kw.list.name'])
+            DID(did=('kw.tuple.scope', 'kw.tuple.name'))
+            DID('arg.scope', name='kwarg.name')
+            DID('arg.name', scope='kwarg.scope')
+        """
+        self.scope: str = ''
+        self.name: str = ''
+
+        did = self._parse_did_from_args(*args, **kwargs)
+
+        self._construct_did(did)
+
+        if not self.is_valid_format():
+            raise DIDError('Object has invalid format after construction: {}'.format(str(self)))
+
+    def _parse_did_from_args(
+            self,
+            *args,
+            **kwargs
+    ) -> Union["DID", str, tuple[str, str], list[str], dict[str, str]]:
+        """
+        Parse the DID object from the given arguments
+        :return: DID object
+        """
+        num_args = len(args)
+        num_kwargs = len(kwargs)
+        if (num_args + num_kwargs) > 2:
+            raise DIDError('Constructor takes at most 2 arguments. Given number: {}'.format(num_args + num_kwargs))
+
+        did: Union["DID", str, tuple[str, str], list[str], dict[str, str]] = ''
+        if num_args == 1:
+            did = args[0]
+
+            if num_kwargs == 1:
+                if isinstance(did, str):
+                    k, v = next(iter(kwargs.items()))
+                    if k == 'scope':
+                        did = (v, did)
+                    elif k == 'name':
+                        did = (did, v)
+                    else:
+                        raise DIDError('Constructor got unexpected keyword argument: {}'.format(k))
+                else:
+                    raise DIDError('First argument of constructor is expected to be string type '
+                                   'when keyword argument is given. Given type: {}'.format(type(did)))
+        elif num_args == 0:
+            did = kwargs.get('did', kwargs)
+        else:
+            did = args
+        return did
+
+    def _construct_did(self, did: Any) -> None:
+        """
+        Construct the DID object from the given input.
+
+        :param did: input to construct the DID object from
+        """
+        if isinstance(did, dict):
+            self._did_from_dict(did)
+        elif isinstance(did, tuple) or isinstance(did, list):
+            self._did_from_list_or_tuple(did)
+        elif isinstance(did, str):
+            self._did_from_str(did)
+        elif isinstance(did, DID):
+            self._did_from_did_object(did)
+        else:
+            raise DIDError('Cannot build object from: {}'.format(type(did)))
+
+        if self.name.endswith('/'):
+            self.name = self.name[:-1]
+
+    def _did_from_str(self, did: str) -> None:
+        """
+        Construct the DID from a string.
+        :param did: string containing the DID information
+        """
+        did_parts = did.split(DID.SCOPE_SEPARATOR, 1)
+        if len(did_parts) == 1:
+            self.name = did
+            self._update_implicit_scope()
+            if not self.has_scope():
+                raise DIDError('Object construction from non-splitable string is ambigious')
+        else:
+            self.scope = did_parts[0]
+            self.name = did_parts[1]
+
+    def _did_from_dict(self, did: dict[str, str]) -> None:
+        """
+        Construct the DID from a dictionary.
+        :param did: dictionary optionally containing the keys 'scope' and 'name'
+        """
+        self.scope = did.get('scope', '')
+        self.name = did.get('name', '')
+        if not self.has_scope():
+            self._update_implicit_scope()
+
+    def _did_from_list_or_tuple(self, did: Union[list[str], tuple[str, str]]) -> None:
+        """
+        Construct the DID from a list or tuple.
+        :param did: list or tuple with expected length of 2
+        """
+        if len(did) != 2:
+            raise DIDError('Construction from tuple or list requires exactly 2 elements. Number of elements passed: %i' % len(did))
+        self.scope = did[0]
+        self.name = did[1]
+
+    def _did_from_did_object(self, did: "DID") -> None:
+        """
+        Construct the DID from another DID object.
+        :param did: DID object
+        """
+        self.scope = did.scope
+        self.name = did.name
+
+    def _update_implicit_scope(self) -> None:
+        """
+        This method sets the scope if it is implicitly given in self.name
+        """
+        did_parts = self.name.split(DID.IMPLICIT_SCOPE_SEPARATOR)
+        num_scope_parts = DID.IMPLICIT_SCOPE_TO_LEN.get(did_parts[0], 0)
+        if num_scope_parts > 0:
+            self.scope = '.'.join(did_parts[0:num_scope_parts])
+
+    def is_valid_format(self) -> bool:
+        """
+        Method to check if the stored DID has a valid format
+        :return: bool
+        """
+        if self.scope.count(DID.SCOPE_SEPARATOR) or self.name.count(DID.SCOPE_SEPARATOR):
+            return False
+        return True
+
+    def has_scope(self) -> bool:
+        """
+        Method to check if the scope part was set
+        :return: bool
+        """
+        return len(self.scope) > 0
+
+    def has_name(self) -> bool:
+        """
+        Method to check if the name part was set
+        :return: bool
+        """
+        return len(self.name) > 0
+
+    def __str__(self) -> str:
+        """
+        Creates the string representation of self
+        :return: string
+        """
+        if self.has_scope() and self.has_name():
+            return '{}{}{}'.format(self.scope, DID.SCOPE_SEPARATOR, self.name)
+        elif self.has_scope():
+            return self.scope
+        return self.name
+
+    def __eq__(self, other: Union[str, "DID"]) -> bool:
+        """
+        Equality comparison with another object
+        :return: bool
+        """
+        if isinstance(other, str):
+            return str(self) == other
+        elif not isinstance(other, DID):
+            try:
+                other = DID(other)
+            except DIDError:
+                return False
+
+        return self.scope == other.scope and self.name == other.name
+
+    def __ne__(self, other: Union[str, "DID"]) -> bool:
+        """
+        Inequality comparison with another object
+        :return: bool
+        """
+        return not self.__eq__(other)
+
+    def __hash__(self) -> int:
+        """
+        Uses the string representation of self to create a hash
+        :return: int
+        """
+        return hash(str(self))