rucio 35.7.0__py3-none-any.whl

rucio/common/constants.py

@@ -0,0 +1,159 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import enum
+from collections import namedtuple
+from typing import Literal, get_args
+
+from rucio.common.config import config_get_bool
+
+"""
+Constants.
+
+"""
+
+RESERVED_KEYS = ['scope', 'name', 'account', 'did_type', 'is_open', 'monotonic', 'obsolete', 'complete',
+                 'availability', 'suppressed', 'bytes', 'length', 'md5', 'adler32', 'rule_evaluation_action',
+                 'rule_evaluation_required', 'expired_at', 'deleted_at', 'created_at', 'updated_at']
+# collection_keys =
+# file_keys =
+
+KEY_TYPES = ['ALL', 'COLLECTION', 'FILE', 'DERIVED']
+# all(container, dataset, file), collection(dataset or container), file, derived(compute from file for collection)
+
+SCHEME_MAP = {'srm': ['srm', 'gsiftp'],
+              'gsiftp': ['srm', 'gsiftp'],
+              'https': ['https', 'davs', 'srm+https', 'cs3s'],
+              'davs': ['https', 'davs', 'srm+https', 'cs3s'],
+              'srm+https': ['https', 'davs', 'srm+https', 'cs3s'],
+              'cs3s': ['https', 'davs', 'srm+https', 'cs3s'],
+              'root': ['root'],
+              'scp': ['scp'],
+              'rsync': ['rsync'],
+              'rclone': ['rclone']}
+if config_get_bool('transfers', 'srm_https_compatibility', raise_exception=False, default=False):
+    SCHEME_MAP['srm'].append('https')
+    SCHEME_MAP['https'].append('srm')
+    SCHEME_MAP['srm'].append('davs')
+    SCHEME_MAP['davs'].append('srm')
+
+SORTING_ALGORITHMS_LITERAL = Literal['geoip', 'closeness', 'custom_table', 'dynamic', 'ranking', 'random']
+SORTING_ALGORITHMS = list(get_args(SORTING_ALGORITHMS_LITERAL))
+
+SUPPORTED_PROTOCOLS_LITERAL = Literal['gsiftp', 'srm', 'root', 'davs', 'http', 'https', 'file', 'storm', 'srm+https', 'scp', 'rsync', 'rclone', 'magnet']
+SUPPORTED_PROTOCOLS: list[str] = list(get_args(SUPPORTED_PROTOCOLS_LITERAL))
+
+RSE_SUPPORTED_PROTOCOL_DOMAINS_LITERAL = Literal['ALL', 'LAN', 'WAN']
+
+RSE_SUPPORTED_PROTOCOL_OPERATIONS_LITERAL = Literal['read', 'write', 'delete', 'third_party_copy_read', 'third_party_copy_write']
+RSE_SUPPORTED_PROTOCOL_OPERATIONS: list[str] = list(get_args(RSE_SUPPORTED_PROTOCOL_OPERATIONS_LITERAL))
+
+FTS_STATE = namedtuple('FTS_STATE', ['SUBMITTED', 'READY', 'ACTIVE', 'FAILED', 'FINISHED', 'FINISHEDDIRTY', 'NOT_USED',
+                                     'CANCELED'])('SUBMITTED', 'READY', 'ACTIVE', 'FAILED', 'FINISHED', 'FINISHEDDIRTY',
+                                                  'NOT_USED', 'CANCELED')
+
+FTS_COMPLETE_STATE = namedtuple('FTS_COMPLETE_STATE', ['OK', 'ERROR'])('Ok', 'Error')
+
+# https://gitlab.cern.ch/fts/fts3/-/blob/master/src/db/generic/Job.h#L41
+FTS_JOB_TYPE = namedtuple('FTS_JOB_TYPE', ['MULTIPLE_REPLICA', 'MULTI_HOP', 'SESSION_REUSE', 'REGULAR'])('R', 'H', 'Y', 'N')
+
+
+# Messages constants
+
+MAX_MESSAGE_LENGTH = 4000
+
+
+class SuspiciousAvailability(enum.Enum):
+    ALL = 0
+    EXIST_COPIES = 1
+    LAST_COPY = 2
+
+
+class ReplicaState(enum.Enum):
+    # From rucio.db.sqla.constants, update that file at the same time as this
+    AVAILABLE = 'A'
+    UNAVAILABLE = 'U'
+    COPYING = 'C'
+    BEING_DELETED = 'B'
+    BAD = 'D'
+    TEMPORARY_UNAVAILABLE = 'T'
+
+
+@enum.unique
+class HermesService(str, enum.Enum):
+    """
+    The services supported by Hermes2.
+    """
+    INFLUX = "INFLUX"
+    ELASTIC = "ELASTIC"
+    EMAIL = "EMAIL"
+    ACTIVEMQ = "ACTIVEMQ"
+
+
+class RseAttr:
+
+    """
+    List of functional RSE attributes.
+
+    This class acts as a namespace containing all RSE attributes referenced in
+    the Rucio source code. Setting them affects Rucio's behaviour in some way.
+    """
+
+    ARCHIVE_TIMEOUT = 'archive_timeout'
+    ASSOCIATED_SITES = 'associated_sites'
+    AUTO_APPROVE_BYTES = 'auto_approve_bytes'
+    AUTO_APPROVE_FILES = 'auto_approve_files'
+    BITTORRENT_TRACKER_ADDR = 'bittorrent_tracker_addr'
+    BLOCK_MANUAL_APPROVAL = 'block_manual_approval'
+    COUNTRY = 'country'
+    DECOMMISSION = 'decommission'
+    DEFAULT_ACCOUNT_LIMIT_BYTES = 'default_account_limit_bytes'
+    FTS = 'fts'
+    GLOBUS_ENDPOINT_ID = 'globus_endpoint_id'
+    GREEDYDELETION = 'greedyDeletion'
+    IS_OBJECT_STORE = 'is_object_store'
+    LFN2PFN_ALGORITHM = 'lfn2pfn_algorithm'
+    MAXIMUM_PIN_LIFETIME = 'maximum_pin_lifetime'
+    MULTIHOP_TOMBSTONE_DELAY = 'multihop_tombstone_delay'
+    NAMING_CONVENTION = 'naming_convention'
+    OIDC_BASE_PATH = 'oidc_base_path'
+    OIDC_SUPPORT = 'oidc_support'
+    PHYSGROUP = 'physgroup'
+    QBITTORRENT_MANAGEMENT_ADDRESS = 'qbittorrent_management_address'
+    RESTRICTED_READ = 'restricted_read'
+    RESTRICTED_WRITE = 'restricted_write'
+    RULE_APPROVERS = 'rule_approvers'
+    S3_URL_STYLE = 's3_url_style'
+    SIGN_URL = 'sign_url'
+    SIMULATE_MULTIRANGE = 'simulate_multirange'
+    SITE = 'site'
+    SKIP_UPLOAD_STAT = 'skip_upload_stat'
+    SOURCE_FOR_TOTAL_SPACE = 'source_for_total_space'
+    SOURCE_FOR_USED_SPACE = 'source_for_used_space'
+    STAGING_BUFFER = 'staging_buffer'
+    STAGING_REQUIRED = 'staging_required'
+    STRICT_COPY = 'strict_copy'
+    TOMBSTONE_DELAY = 'tombstone_delay'
+    TYPE = 'type'
+    USE_IPV4 = 'use_ipv4'
+    VERIFY_CHECKSUM = 'verify_checksum'
+
+    # The following RSE attributes are exclusively used in the permission layer
+    # and are likely VO-specific.
+
+    BLOCK_MANUAL_APPROVE = 'block_manual_approve'
+    CMS_TYPE = 'cms_type'
+    DEFAULT_LIMIT_FILES = 'default_limit_files'
+    QUOTA_APPROVERS = 'quota_approvers'
+    RULE_DELETERS = 'rule_deleters'

rucio/common/constraints.py

@@ -0,0 +1,17 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# List of authorized value types for key
+AUTHORIZED_VALUE_TYPES = (float, int, str)
+STRING_TYPES = (str, )

rucio/common/didtype.py

@@ -0,0 +1,189 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+DID type to represent a did and to simplify operations on it
+"""
+
+from typing import Union
+
+from rucio.common.exception import DIDError
+
+
+class DID:
+
+    """
+    Class used to store a DID
+    Given an object did of type DID
+    scope is stored in did.scope
+    name is stored in did.name
+    """
+
+    SCOPE_SEPARATOR = ':'
+    IMPLICIT_SCOPE_SEPARATOR = '.'
+    IMPLICIT_SCOPE_TO_LEN = {'user': 2, 'group': 2}
+    __slots__ = ['scope', 'name']
+
+    def __init__(self, *args, **kwargs):
+        """
+        Constructs the DID object. Possible parameter combinations are:
+            DID()
+            DID('scope:name.did.str')
+            DID('user.implicit.scope.in.name')
+            DID('custom.scope', 'custom.name')
+            DID(['list.scope', 'list.name'])
+            DID(('tuple.scope', 'tuple.name'))
+            DID({'scope': 'dict.scope', 'name': 'dict.name'})
+            DID(scope='kw.scope')
+            DID(name='kw.name')
+            DID(name='user.kw.implicit.scope')
+            DID(scope='kw.scope', name='kw.name')
+            DID(did={'scope': 'kw.did.scope', 'name': 'kw.did.name'})
+            DID(did=['kw.list.scope', 'kw.list.name'])
+            DID(did=('kw.tuple.scope', 'kw.tuple.name'))
+            DID('arg.scope', name='kwarg.name')
+            DID('arg.name', scope='kwarg.scope')
+        """
+        self.scope: str = ''
+        self.name: str = ''
+
+        num_args = len(args)
+        num_kwargs = len(kwargs)
+        if (num_args + num_kwargs) > 2:
+            raise DIDError('Constructor takes at most 2 arguments. Given number: {}'.format(num_args + num_kwargs))
+
+        did: Union["DID", str, tuple[str, str], list[str], dict[str, str]] = ''
+        if num_args == 1:
+            did = args[0]
+
+            if num_kwargs == 1:
+                if isinstance(did, str):
+                    k, v = next(iter(kwargs.items()))
+                    if k == 'scope':
+                        did = (v, did)
+                    elif k == 'name':
+                        did = (did, v)
+                    else:
+                        raise DIDError('Constructor got unexpected keyword argument: {}'.format(k))
+                else:
+                    raise DIDError('First argument of constructor is expected to be string type'
+                                   'when keyword argument is given. Given type: {}'.format(type(did)))
+        elif num_args == 0:
+            did = kwargs.get('did', kwargs)
+        else:
+            did = args
+
+        if isinstance(did, dict):
+            self.scope = did.get('scope', '')
+            self.name = did.get('name', '')
+            if not self.has_scope():
+                self.update_implicit_scope()
+        elif isinstance(did, tuple) or isinstance(did, list):
+            if len(did) != 2:
+                raise DIDError('Construction from tuple or list requires exactly 2 elements')
+            self.scope = did[0]
+            self.name = did[1]
+        elif isinstance(did, str):
+            did_parts = did.split(DID.SCOPE_SEPARATOR, 1)
+            if len(did_parts) == 1:
+                self.name = did
+                self.update_implicit_scope()
+                if not self.has_scope():
+                    raise DIDError('Object construction from non-splitable string is ambigious')
+            else:
+                self.scope = did_parts[0]
+                self.name = did_parts[1]
+        elif isinstance(did, DID):
+            self.scope = did.scope
+            self.name = did.name
+        else:
+            raise DIDError('Cannot build object from: {}'.format(type(did)))
+
+        if self.name.endswith('/'):
+            self.name = self.name[:-1]
+
+        if not self.is_valid_format():
+            raise DIDError('Object has invalid format after construction: {}'.format(str(self)))
+
+    def update_implicit_scope(self) -> None:
+        """
+        This method sets the scope if it is implicitly given in self.name
+        """
+        did_parts = self.name.split(DID.IMPLICIT_SCOPE_SEPARATOR)
+        num_scope_parts = DID.IMPLICIT_SCOPE_TO_LEN.get(did_parts[0], 0)
+        if num_scope_parts > 0:
+            self.scope = '.'.join(did_parts[0:num_scope_parts])
+
+    def is_valid_format(self) -> bool:
+        """
+        Method to check if the stored DID has a valid format
+        :return: bool
+        """
+        if self.scope.count(DID.SCOPE_SEPARATOR) or self.name.count(DID.SCOPE_SEPARATOR):
+            return False
+        return True
+
+    def has_scope(self) -> bool:
+        """
+        Method to check if the scope part was set
+        :return: bool
+        """
+        return len(self.scope) > 0
+
+    def has_name(self) -> bool:
+        """
+        Method to check if the name part was set
+        :return: bool
+        """
+        return len(self.name) > 0
+
+    def __str__(self) -> str:
+        """
+        Creates the string representation of self
+        :return: string
+        """
+        if self.has_scope() and self.has_name():
+            return '{}{}{}'.format(self.scope, DID.SCOPE_SEPARATOR, self.name)
+        elif self.has_scope():
+            return self.scope
+        return self.name
+
+    def __eq__(self, other: Union[str, "DID"]) -> bool:
+        """
+        Equality comparison with another object
+        :return: bool
+        """
+        if isinstance(other, str):
+            return str(self) == other
+        elif not isinstance(other, DID):
+            try:
+                other = DID(other)
+            except DIDError:
+                return False
+
+        return self.scope == other.scope and self.name == other.name
+
+    def __ne__(self, other: Union[str, "DID"]) -> bool:
+        """
+        Inequality comparison with another object
+        :return: bool
+        """
+        return not self.__eq__(other)
+
+    def __hash__(self) -> int:
+        """
+        Uses the string representation of self to create a hash
+        :return: int
+        """
+        return hash(str(self))

rucio/common/dumper/__init__.py

@@ -0,0 +1,335 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import bz2
+import contextlib
+import datetime
+import gzip
+import logging
+import os
+import re
+import sys
+import tempfile
+from typing import TYPE_CHECKING, Any, Literal, Optional, Union
+
+import gfal2
+import magic
+import requests
+
+from rucio.common import config
+from rucio.core.rse import get_rse_id, get_rse_protocols
+
+if TYPE_CHECKING:
+    from collections.abc import Iterator
+    from multiprocessing.connection import Connection
+    from types import ModuleType
+    from typing import IO, TextIO
+
+    from _typeshed import FileDescriptorOrPath, GenericPath, StrOrBytesPath
+
+    from rucio.common.types import RSEProtocolDict
+
+
+class HTTPDownloadFailed(Exception):
+    def __init__(self, msg: str = '', code: Optional[str] = None):
+        self.code = code
+        if code is not None:
+            msg = '{0} (Status {1})'.format(msg, code)
+        super(HTTPDownloadFailed, self).__init__(msg)
+
+
+class LogPipeHandler(logging.Handler):
+    def __init__(self, pipe: "Connection"):
+        super(LogPipeHandler, self).__init__()
+        self.pipe = pipe
+
+    def emit(self, record: logging.LogRecord) -> None:
+        self.pipe.send(self.format(record))
+
+    def close(self) -> None:
+        super(LogPipeHandler, self).close()
+        self.pipe.close()
+
+
+def error(text: str, exit_code: int = 1) -> None:
+    '''
+    Log and print `text` error. This function ends the execution of the program with exit code
+    `exit_code` (defaults to 1).
+    '''
+    logger = logging.getLogger('dumper.__init__')
+    logger.error(text)
+    sys.stderr.write(text + '\n')
+    exit(1)
+
+
+def mkdir(dir_: "StrOrBytesPath") -> None:
+    '''
+    This functions creates the `dir_` directory if it doesn't exist. If `dir_`
+    already exists this function does nothing.
+    '''
+    try:
+        os.mkdir(dir_)
+    except OSError as error:
+        if error.errno != 17:
+            raise error
+
+
+def cacert_config(config: "ModuleType", rucio_home: str) -> Optional[Union["FileDescriptorOrPath", Literal[False]]]:
+    logger = logging.getLogger('dumper.__init__')
+    try:
+        cacert = config.config_get('client', 'ca_cert').replace('$RUCIO_HOME', rucio_home)
+    except KeyError:
+        cacert = None
+
+    if cacert is None or not os.path.exists(cacert):
+        logger.warning('Configured CA Certificate file "%s" not found: Host certificate verification disabled', cacert)
+        cacert = False
+
+    return cacert
+
+
+def rucio_home() -> str:
+    return os.environ.get('RUCIO_HOME', '/opt/rucio')
+
+
+def get_requests_session() -> requests.Session:
+    requests_session = requests.Session()
+    requests_session.verify = cacert_config(config, rucio_home())  # type: ignore
+    requests_session.stream = True
+    return requests_session
+
+
+DUMPS_CACHE_DIR = 'cache'
+RESULTS_DIR = 'results'
+CHUNK_SIZE = 4194304  # 4MiB
+
+
+# There are two Python modules with the name `magic`, luckily both do
+# the same thing.
+# pylint: disable=no-member
+if 'open' in dir(magic):
+    _mime = magic.open(magic.MAGIC_MIME)
+    _mime.load()
+    mimetype = _mime.file
+else:
+    mimetype = lambda filename: magic.from_file(filename, mime=True)  # NOQA
+# pylint: enable=no-member
+
+
+def isplaintext(filename: "GenericPath") -> bool:
+    '''
+    Returns True if `filename` has mimetype == 'text/plain'.
+    '''
+    if os.path.islink(filename):
+        filename = os.readlink(filename)
+    return mimetype(filename).split(';')[0] == 'text/plain'
+
+
+def smart_open(filename: "GenericPath") -> Optional[Union["TextIO", gzip.GzipFile]]:
+    '''
+    Returns an open file object if `filename` is plain text, else assumes
+    it is a bzip2 compressed file and returns a file-like object to
+    handle it.
+    '''
+    f = None
+    if isplaintext(filename):
+        f = open(filename, 'rt')
+    else:
+        file_type = mimetype(filename)
+        if file_type.find('gzip') > -1:
+            f = gzip.GzipFile(filename, 'rt')
+        elif file_type.find('bzip2') > -1:
+            f = bz2.open(filename, 'rt')
+        else:
+            pass  # Not supported format
+    return f
+
+
+@contextlib.contextmanager
+def temp_file(
+    directory: str,
+    final_name: Optional[str] = None,
+    binary: bool = False
+) -> "Iterator[tuple[IO[Any], StrOrBytesPath]]":
+    '''
+    Allows to create a temporal file to store partial results, when the
+    file is complete it is renamed to `final_name`.
+
+    - `directory`: working path to create the temporal and the final file.
+    - `final_name`: Path of the final file, relative to `directory`.
+       If the `final_name` is omitted or None the renaming step is omitted,
+       leaving the temporal file with the results.
+    - `binary`: whether to open the file in binary mode (default: False).
+
+    Important: `directory` and `final_name` must be in the same filesystem as
+    a hardlink is used to rename the temporal file.
+
+    Example:
+    >>> with temp_file('/tmp', final_name='b') as (f, fname):
+    >>>     print('The temporary file is named', fname)
+    >>>     f.write('x')
+    >>> assert not os.path.exist('/tmp/' + fname)
+    >>> assert os.path.exist('/tmp/b')
+    '''
+    logger = logging.getLogger('dumper.__init__')
+
+    fd, tpath = tempfile.mkstemp(dir=directory)
+    tmp = os.fdopen(fd, 'wb' if binary else 'w')
+
+    try:
+        yield tmp, os.path.basename(tpath)
+    except:
+        # Close and remove temporal file on failure
+        tmp.close()
+        os.unlink(tpath)
+        raise
+
+    tmp.close()
+    if final_name is not None:
+        dst_path = os.path.join(directory, final_name)
+        logger.debug('Renaming "%s" to "%s"', tpath, dst_path)
+        os.link(tpath, dst_path)
+        os.unlink(tpath)
+
+
+DATETIME_FORMAT = '%Y-%m-%d %H:%M:%S'
+DATETIME_FORMAT_FULL = '%Y-%m-%dT%H:%M:%S'
+MILLISECONDS_RE = re.compile(r'\.(\d{3})Z$')
+
+
+def to_datetime(str_or_datetime: Union[datetime.datetime, str]) -> Optional[datetime.datetime]:
+    """
+    Convert string to datetime. The format is somewhat flexible.
+    Timezone information is ignored.
+    """
+    logger = logging.getLogger('dumper.__init__')
+    if isinstance(str_or_datetime, datetime.datetime):
+        return str_or_datetime
+    elif str_or_datetime.strip() == '':
+        return None
+
+    str_or_datetime = str_or_datetime.replace('T', ' ')
+    try:
+        logger.debug(
+            'Trying to parse "%s" date with '
+            'resolution of seconds or tenths of seconds',
+            str_or_datetime,
+        )
+        str_or_datetime = re.sub(r'\.\d$', '', str_or_datetime)
+        date = datetime.datetime.strptime(
+            str_or_datetime,
+            DATETIME_FORMAT,
+        )
+    except ValueError:
+        logger.debug(
+            'Trying to parse "%s" date with resolution of milliseconds',
+            str_or_datetime,
+        )
+        milliseconds = int(MILLISECONDS_RE.search(str_or_datetime).group(1))
+        str_or_datetime = MILLISECONDS_RE.sub('', str_or_datetime)
+        date = datetime.datetime.strptime(
+            str_or_datetime,
+            DATETIME_FORMAT,
+        )
+        date = date + datetime.timedelta(microseconds=milliseconds * 1000)
+    return date
+
+
+def ddmendpoint_preferred_protocol(ddmendpoint: str) -> "RSEProtocolDict":
+    return next(p for p in get_rse_protocols(get_rse_id(ddmendpoint))['protocols'] if p['domains']['wan']['read'] == 1)
+
+
+def ddmendpoint_url(ddmendpoint: str) -> str:
+    preferred_protocol = ddmendpoint_preferred_protocol(ddmendpoint)
+    prefix = re.sub(r'rucio/$', '', preferred_protocol['prefix'])
+    return '{scheme}://{hostname}:{port}'.format(**preferred_protocol) + prefix
+
+
+def http_download_to_file(url: str, file_: "IO", session: Optional[requests.Session] = None) -> None:
+    '''
+    Download the file in `url` storing it in the `file_` file-like
+    object.
+    If given `session` must be a requests.Session instance, and will be
+    used to download the file, otherwise requests.get() will be used.
+    '''
+    def _do_download(url, file_, session, try_decode=False):
+        if session is None:
+            response = requests.get(url, stream=True)
+        else:
+            response = session.get(url)
+
+        if response.status_code != 200:
+            logging.error(
+                'Retrieving %s returned %d status code',
+                url,
+                response.status_code,
+            )
+            raise HTTPDownloadFailed('Error downloading ' + url, str(response.status_code))
+
+        if try_decode:
+            if response.encoding is None:
+                response.encoding = 'utf-8'
+            for chunk in response.iter_content(CHUNK_SIZE, decode_unicode=True):
+                file_.write(chunk)
+        else:
+            for chunk in response.iter_content(CHUNK_SIZE):
+                file_.write(chunk)
+
+    try:
+        # try without decoding first
+        _do_download(url, file_, session, False)
+    except TypeError:
+        # if that fails due to writing binary data to text file, try to force decode
+        _do_download(url, file_, session, True)
+
+
+def http_download(url: str, filename: "FileDescriptorOrPath") -> None:
+    '''
+    Download the file in `url` storing it in the path given by `filename`.
+    '''
+    with open(filename, 'w') as f:
+        http_download_to_file(url, f)
+
+
+def gfal_download_to_file(url: str, file_: "IO") -> None:
+    '''
+    Download the file in `url` storing it in the `file_` file-like
+    object.
+    '''
+    logger = logging.getLogger('dumper.__init__')
+    ctx = gfal2.creat_context()  # pylint: disable=no-member
+    infile = ctx.open(url, 'r')
+
+    try:
+        chunk = infile.read(CHUNK_SIZE)
+    except gfal2.GError as e:
+        if e.code == 70:
+            logger.debug('GError(70) raised, using GRIDFTP PLUGIN:STAT_ON_OPEN=False workaround to download %s', url)
+            ctx.set_opt_boolean('GRIDFTP PLUGIN', 'STAT_ON_OPEN', False)
+            infile = ctx.open(url, 'r')
+            chunk = infile.read(CHUNK_SIZE)
+        else:
+            raise
+
+    while chunk:
+        file_.write(chunk)
+        chunk = infile.read(CHUNK_SIZE)
+
+
+def gfal_download(url: str, filename: "FileDescriptorOrPath") -> None:
+    '''
+    Download the file in `url` storing it in the path given by `filename`.
+    '''
+    with open(filename, 'w') as f:
+        gfal_download_to_file(url, f)