rucio 32.8.6__py3-none-any.whl

rucio/common/constants.py

@@ -0,0 +1,92 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import enum
+from collections import namedtuple
+
+from rucio.common.config import config_get_bool
+
+"""
+Constants.
+
+"""
+
+RESERVED_KEYS = ['scope', 'name', 'account', 'did_type', 'is_open', 'monotonic', 'obsolete', 'complete',
+                 'availability', 'suppressed', 'bytes', 'length', 'md5', 'adler32', 'rule_evaluation_action',
+                 'rule_evaluation_required', 'expired_at', 'deleted_at', 'created_at', 'updated_at']
+# collection_keys =
+# file_keys =
+
+KEY_TYPES = ['ALL', 'COLLECTION', 'FILE', 'DERIVED']
+# all(container, dataset, file), collection(dataset or container), file, derived(compute from file for collection)
+
+SCHEME_MAP = {'srm': ['srm', 'gsiftp'],
+              'gsiftp': ['srm', 'gsiftp'],
+              'https': ['https', 'davs', 'srm+https', 'cs3s'],
+              'davs': ['https', 'davs', 'srm+https', 'cs3s'],
+              'srm+https': ['https', 'davs', 'srm+https', 'cs3s'],
+              'cs3s': ['https', 'davs', 'srm+https', 'cs3s'],
+              'root': ['root'],
+              'scp': ['scp'],
+              'rsync': ['rsync'],
+              'rclone': ['rclone']}
+if config_get_bool('transfers', 'srm_https_compatibility', raise_exception=False, default=False):
+    SCHEME_MAP['srm'].append('https')
+    SCHEME_MAP['https'].append('srm')
+    SCHEME_MAP['srm'].append('davs')
+    SCHEME_MAP['davs'].append('srm')
+
+SUPPORTED_PROTOCOLS = ['gsiftp', 'srm', 'root', 'davs', 'http', 'https', 'file', 'storm', 'srm+https', 'scp', 'rsync', 'rclone']
+
+FTS_STATE = namedtuple('FTS_STATE', ['SUBMITTED', 'READY', 'ACTIVE', 'FAILED', 'FINISHED', 'FINISHEDDIRTY', 'NOT_USED',
+                                     'CANCELED'])('SUBMITTED', 'READY', 'ACTIVE', 'FAILED', 'FINISHED', 'FINISHEDDIRTY',
+                                                  'NOT_USED', 'CANCELED')
+
+FTS_COMPLETE_STATE = namedtuple('FTS_COMPLETE_STATE', ['OK', 'ERROR'])('Ok', 'Error')
+
+# https://gitlab.cern.ch/fts/fts3/-/blob/master/src/db/generic/Job.h#L41
+FTS_JOB_TYPE = namedtuple('FTS_JOB_TYPE', ['MULTIPLE_REPLICA', 'MULTI_HOP', 'SESSION_REUSE', 'REGULAR'])('R', 'H', 'Y', 'N')
+
+
+# Messages constants
+
+MAX_MESSAGE_LENGTH = 4000
+
+
+class SuspiciousAvailability(enum.Enum):
+    ALL = 0
+    EXIST_COPIES = 1
+    LAST_COPY = 2
+
+
+class ReplicaState(enum.Enum):
+    # From rucio.db.sqla.constants, update that file at the same time as this
+    AVAILABLE = 'A'
+    UNAVAILABLE = 'U'
+    COPYING = 'C'
+    BEING_DELETED = 'B'
+    BAD = 'D'
+    TEMPORARY_UNAVAILABLE = 'T'
+
+
+@enum.unique
+class HermesService(str, enum.Enum):
+    """
+    The services supported by Hermes2.
+    """
+    INFLUX = "INFLUX"
+    ELASTIC = "ELASTIC"
+    EMAIL = "EMAIL"
+    ACTIVEMQ = "ACTIVEMQ"

rucio/common/constraints.py

@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# List of authorized value types for key
+AUTHORIZED_VALUE_TYPES = (float, int, str)
+STRING_TYPES = (str, )

rucio/common/didtype.py

@@ -0,0 +1,187 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+DID type to represent a did and to simplify operations on it
+"""
+
+from rucio.common.exception import DIDError
+
+
+class DID(object):
+
+    """
+    Class used to store a DID
+    Given an object did of type DID
+    scope is stored in did.scope
+    name is stored in did.name
+    """
+
+    SCOPE_SEPARATOR = ':'
+    IMPLICIT_SCOPE_SEPARATOR = '.'
+    IMPLICIT_SCOPE_TO_LEN = {'user': 2, 'group': 2}
+    __slots__ = ['scope', 'name']
+
+    def __init__(self, *args, **kwargs):
+        """
+        Constructs the DID object. Possible parameter combinations are:
+            DID()
+            DID('scope:name.did.str')
+            DID('user.implicit.scope.in.name')
+            DID('custom.scope', 'custom.name')
+            DID(['list.scope', 'list.name'])
+            DID(('tuple.scope', 'tuple.name'))
+            DID({'scope': 'dict.scope', 'name': 'dict.name'})
+            DID(scope='kw.scope')
+            DID(name='kw.name')
+            DID(name='user.kw.implicit.scope')
+            DID(scope='kw.scope', name='kw.name')
+            DID(did={'scope': 'kw.did.scope', 'name': 'kw.did.name'})
+            DID(did=['kw.list.scope', 'kw.list.name'])
+            DID(did=('kw.tuple.scope', 'kw.tuple.name'))
+            DID('arg.scope', name='kwarg.name')
+            DID('arg.name', scope='kwarg.scope')
+        """
+        self.scope = self.name = ''
+
+        num_args = len(args)
+        num_kwargs = len(kwargs)
+        if (num_args + num_kwargs) > 2:
+            raise DIDError('Constructor takes at most 2 arguments. Given number: {}'.format(num_args + num_kwargs))
+
+        did = ''
+        if num_args == 1:
+            did = args[0]
+
+            if num_kwargs == 1:
+                if not isinstance(did, str):
+                    raise DIDError('First argument of constructor is expected to be string type'
+                                   'when keyword argument is given. Given type: {}'.format(type(did)))
+
+                k, v = next(iter(kwargs.items()))
+                if k == 'scope':
+                    did = (v, did)
+                elif k == 'name':
+                    did = (did, v)
+                else:
+                    raise DIDError('Constructor got unexpected keyword argument: {}'.format(k))
+        elif num_args == 0:
+            did = kwargs.get('did', kwargs)
+        else:
+            did = args
+
+        if isinstance(did, dict):
+            self.scope = did.get('scope', '')
+            self.name = did.get('name', '')
+            if not self.has_scope():
+                self.update_implicit_scope()
+        elif isinstance(did, tuple) or isinstance(did, list):
+            if len(did) != 2:
+                raise DIDError('Construction from tuple or list requires exactly 2 elements')
+            self.scope = did[0]
+            self.name = did[1]
+        elif isinstance(did, str):
+            did_parts = did.split(DID.SCOPE_SEPARATOR, 1)
+            if len(did_parts) == 1:
+                self.name = did
+                self.update_implicit_scope()
+                if not self.has_scope():
+                    raise DIDError('Object construction from non-splitable string is ambigious')
+            else:
+                self.scope = did_parts[0]
+                self.name = did_parts[1]
+        elif isinstance(did, DID):
+            self.scope = did.scope
+            self.name = did.name
+        else:
+            raise DIDError('Cannot build object from: {}'.format(type(did)))
+
+        if self.name.endswith('/'):
+            self.name = self.name[:-1]
+
+        if not self.is_valid_format():
+            raise DIDError('Object has invalid format after construction: {}'.format(str(self)))
+
+    def update_implicit_scope(self):
+        """
+        This method sets the scope  if it is implicitly given in self.name
+        """
+        did_parts = self.name.split(DID.IMPLICIT_SCOPE_SEPARATOR)
+        num_scope_parts = DID.IMPLICIT_SCOPE_TO_LEN.get(did_parts[0], 0)
+        if num_scope_parts > 0:
+            self.scope = '.'.join(did_parts[0:num_scope_parts])
+
+    def is_valid_format(self):
+        """
+        Method to check if the stored DID has a valid format
+        :return: bool
+        """
+        if self.scope.count(DID.SCOPE_SEPARATOR) or self.name.count(DID.SCOPE_SEPARATOR):
+            return False
+        return True
+
+    def has_scope(self):
+        """
+        Method to check if the scope part was set
+        :return: bool
+        """
+        return len(self.scope) > 0
+
+    def has_name(self):
+        """
+        Method to check if the name part was set
+        :return: bool
+        """
+        return len(self.name) > 0
+
+    def __str__(self):
+        """
+        Creates the string representation of self
+        :return: string
+        """
+        if self.has_scope() and self.has_name():
+            return '{}{}{}'.format(self.scope, DID.SCOPE_SEPARATOR, self.name)
+        elif self.has_scope():
+            return self.scope
+        return self.name
+
+    def __eq__(self, other):
+        """
+        Equality comparison with another object
+        :return: bool
+        """
+        if isinstance(other, str):
+            return str(self) == other
+        elif not isinstance(other, DID):
+            try:
+                other = DID(other)
+            except DIDError:
+                return False
+
+        return self.scope == other.scope and self.name == other.name
+
+    def __ne__(self, other):
+        """
+        Inequality comparison with another object
+        :return: bool
+        """
+        return not self.__eq__(other)
+
+    def __hash__(self):
+        """
+        Uses the string representation of self to create a hash
+        :return: int
+        """
+        return hash(str(self))

rucio/common/dumper/__init__.py

@@ -0,0 +1,306 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import bz2
+import contextlib
+import datetime
+import gzip
+import logging
+import os
+import re
+import sys
+import tempfile
+
+import gfal2
+import magic
+import requests
+
+from rucio.common import config
+from rucio.core.rse import get_rse_id, get_rse_protocols
+
+
+class HTTPDownloadFailed(Exception):
+    def __init__(self, msg='', code=None):
+        self.code = code
+        if code is not None:
+            msg = '{0} (Status {1})'.format(msg, code)
+        super(HTTPDownloadFailed, self).__init__(msg)
+
+
+class LogPipeHandler(logging.Handler, object):
+    def __init__(self, pipe):
+        super(LogPipeHandler, self).__init__()
+        self.pipe = pipe
+
+    def emit(self, record):
+        self.pipe.send(self.format(record))
+
+    def close(self):
+        super(LogPipeHandler, self).close()
+        self.pipe.close()
+
+
+def error(text, exit_code=1):
+    '''
+    Log and print `text` error. This function ends the execution of the program with exit code
+    `exit_code` (defaults to 1).
+    '''
+    logger = logging.getLogger('dumper.__init__')
+    logger.error(text)
+    sys.stderr.write(text + '\n')
+    exit(1)
+
+
+def mkdir(dir_):
+    '''
+    This functions creates the `dir_` directory if it doesn't exist. If `dir_`
+    already exists this function does nothing.
+    '''
+    try:
+        os.mkdir(dir_)
+    except OSError as error:
+        assert error.errno == 17
+
+
+def cacert_config(config, rucio_home):
+    logger = logging.getLogger('dumper.__init__')
+    try:
+        cacert = config.config_get('client', 'ca_cert').replace('$RUCIO_HOME', rucio_home)
+    except KeyError:
+        cacert = None
+
+    if cacert is None or not os.path.exists(cacert):
+        logger.warning('Configured CA Certificate file "%s" not found: Host certificate verification disabled', cacert)
+        cacert = False
+
+    return cacert
+
+
+def rucio_home():
+    return os.environ.get('RUCIO_HOME', '/opt/rucio')
+
+
+def get_requests_session():
+    requests_session = requests.Session()
+    requests_session.verify = cacert_config(config, rucio_home())
+    requests_session.stream = True
+    return requests_session
+
+
+DUMPS_CACHE_DIR = 'cache'
+RESULTS_DIR = 'results'
+CHUNK_SIZE = 4194304  # 4MiB
+
+
+# There are two Python modules with the name `magic`, luckily both do
+# the same thing.
+# pylint: disable=no-member
+if 'open' in dir(magic):
+    _mime = magic.open(magic.MAGIC_MIME)
+    _mime.load()
+    mimetype = _mime.file
+else:
+    mimetype = lambda filename: magic.from_file(filename, mime=True)  # NOQA
+# pylint: enable=no-member
+
+
+def isplaintext(filename):
+    '''
+    Returns True if `filename` has mimetype == 'text/plain'.
+    '''
+    if os.path.islink(filename):
+        filename = os.readlink(filename)
+    return mimetype(filename).split(';')[0] == 'text/plain'
+
+
+def smart_open(filename):
+    '''
+    Returns an open file object if `filename` is plain text, else assumes
+    it is a bzip2 compressed file and returns a file-like object to
+    handle it.
+    '''
+    f = None
+    if isplaintext(filename):
+        f = open(filename, 'rt')
+    else:
+        file_type = mimetype(filename)
+        if file_type.find('gzip') > -1:
+            f = gzip.GzipFile(filename, 'rt')
+        elif file_type.find('bzip2') > -1:
+            f = bz2.open(filename, 'rt')
+        else:
+            pass  # Not supported format
+    return f
+
+
+@contextlib.contextmanager
+def temp_file(directory, final_name=None, binary=False):
+    '''
+    Allows to create a temporal file to store partial results, when the
+    file is complete it is renamed to `final_name`.
+
+    - `directory`: working path to create the temporal and the final file.
+    - `final_name`: Path of the final file, relative to `directory`.
+       If the `final_name` is omitted or None the renaming step is ommited,
+       leaving the temporal file with the results.
+    - `binary`: whether to open the file in binary mode (default: False).
+
+    Important: `directory` and `final_name` must be in the same filesystem as
+    a hardlink is used to rename the temporal file.
+
+    Example:
+    >>> with temp_file('/tmp', final_name='b') as (f, fname):
+    >>>     print('The temporary file is named', fname)
+    >>>     f.write('x')
+    >>> assert not os.path.exist('/tmp/' + fname)
+    >>> assert os.path.exist('/tmp/b')
+    '''
+    logger = logging.getLogger('dumper.__init__')
+
+    fd, tpath = tempfile.mkstemp(dir=directory)
+    tmp = os.fdopen(fd, 'wb' if binary else 'w')
+
+    try:
+        yield tmp, os.path.basename(tpath)
+    except:
+        # Close and remove temporal file on failure
+        tmp.close()
+        os.unlink(tpath)
+        raise
+
+    tmp.close()
+    if final_name is not None:
+        dst_path = os.path.join(directory, final_name)
+        logger.debug('Renaming "%s" to "%s"', tpath, dst_path)
+        os.link(tpath, dst_path)
+        os.unlink(tpath)
+
+
+DATETIME_FORMAT = '%Y-%m-%d %H:%M:%S'
+DATETIME_FORMAT_FULL = '%Y-%m-%dT%H:%M:%S'
+MILLISECONDS_RE = re.compile(r'\.(\d{3})Z$')
+
+
+def to_datetime(str_or_datetime):
+    """
+    Convert string to datetime. The format is somewhat flexible.
+    Timezone information is ignored.
+    """
+    logger = logging.getLogger('dumper.__init__')
+    if isinstance(str_or_datetime, datetime.datetime):
+        return str_or_datetime
+    elif str_or_datetime.strip() == '':
+        return None
+
+    str_or_datetime = str_or_datetime.replace('T', ' ')
+    try:
+        logger.debug(
+            'Trying to parse "%s" date with '
+            'resolution of seconds or tenths of seconds',
+            str_or_datetime,
+        )
+        str_or_datetime = re.sub(r'\.\d$', '', str_or_datetime)
+        date = datetime.datetime.strptime(
+            str_or_datetime,
+            DATETIME_FORMAT,
+        )
+    except ValueError:
+        logger.debug(
+            'Trying to parse "%s" date with resolution of milliseconds',
+            str_or_datetime,
+        )
+        miliseconds = int(MILLISECONDS_RE.search(str_or_datetime).group(1))
+        str_or_datetime = MILLISECONDS_RE.sub('', str_or_datetime)
+        date = datetime.datetime.strptime(
+            str_or_datetime,
+            DATETIME_FORMAT,
+        )
+        date = date + datetime.timedelta(microseconds=miliseconds * 1000)
+    return date
+
+
+def ddmendpoint_preferred_protocol(ddmendpoint):
+    return next(p for p in get_rse_protocols(get_rse_id(ddmendpoint))['protocols'] if p['domains']['wan']['read'] == 1)
+
+
+def ddmendpoint_url(ddmendpoint):
+    preferred_protocol = ddmendpoint_preferred_protocol(ddmendpoint)
+    prefix = re.sub(r'rucio/$', '', preferred_protocol['prefix'])
+    return '{scheme}://{hostname}:{port}'.format(**preferred_protocol) + prefix
+
+
+def http_download_to_file(url, file_, session=None):
+    '''
+    Download the file in `url` storing it in the `file_` file-like
+    object.
+    If given `session` must be a requests.Session instance, and will be
+    used to download the file, otherwise requests.get() will be used.
+    '''
+    if session is None:
+        response = requests.get(url, stream=True)
+    else:
+        response = session.get(url)
+
+    if response.status_code != 200:
+        logging.error(
+            'Retrieving %s returned %d status code',
+            url,
+            response.status_code,
+        )
+        raise HTTPDownloadFailed('Error downloading ' + url, response.status_code)
+
+    for chunk in response.iter_content(CHUNK_SIZE):
+        file_.write(chunk)
+
+
+def http_download(url, filename):
+    '''
+    Download the file in `url` storing it in the path given by `filename`.
+    '''
+    with open(filename, 'w') as f:
+        http_download_to_file(url, f)
+
+
+def gfal_download_to_file(url, file_):
+    '''
+    Download the file in `url` storing it in the `file_` file-like
+    object.
+    '''
+    logger = logging.getLogger('dumper.__init__')
+    ctx = gfal2.creat_context()  # pylint: disable=no-member
+    infile = ctx.open(url, 'r')
+
+    try:
+        chunk = infile.read(CHUNK_SIZE)
+    except gfal2.GError as e:
+        if e.code == 70:
+            logger.debug('GError(70) raised, using GRIDFTP PLUGIN:STAT_ON_OPEN=False workarround to download %s', url)
+            ctx.set_opt_boolean('GRIDFTP PLUGIN', 'STAT_ON_OPEN', False)
+            infile = ctx.open(url, 'r')
+            chunk = infile.read(CHUNK_SIZE)
+        else:
+            raise
+
+    while chunk:
+        file_.write(chunk)
+        chunk = infile.read(CHUNK_SIZE)
+
+
+def gfal_download(url, filename):
+    '''
+    Download the file in `url` storing it in the path given by `filename`.
+    '''
+    with open(filename, 'w') as f:
+        gfal_download_to_file(url, f)