rucio-clients 35.7.0__py3-none-any.whl → 37.0.0rc2__py3-none-any.whl

rucio/common/bittorrent.py

@@ -0,0 +1,234 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import copy
+import hashlib
+import itertools
+import math
+import os
+import time
+from typing import TYPE_CHECKING, Any, Optional, Union
+
+from rucio.common.exception import RucioException
+
+if TYPE_CHECKING:
+    from _typeshed import FileDescriptorOrPath
+
+
+def _next_pow2(num: int) -> int:
+    if not num:
+        return 0
+    return math.ceil(math.log2(num))
+
+
+def _bittorrent_v2_piece_length_pow2(file_size: int) -> int:
+    """
+    Automatically chooses the `piece size` so that `piece layers`
+    is kept small(er) than usually. This is a balancing act:
+    having a big piece_length requires more work on bittorrent client
+    side to validate hashes, but having it small requires more
+    place to store the `piece layers` in the database.
+
+    Returns the result as the exponent 'x' for power of 2.
+    To get the actual length in bytes, the caller should compute 2^x.
+    """
+
+    # by the bittorrent v2 specification, the piece size is equal to block size = 16KiB
+    min_piece_len_pow2 = 14  # 2 ** 14 == 16 KiB
+    if not file_size:
+        return min_piece_len_pow2
+    # Limit the maximum size of pieces_layers hash chain for bittorrent v2,
+    # because we'll have to store it in the database
+    max_pieces_layers_size_pow2 = 20  # 2 ** 20 == 1 MiB
+    # sha256 requires 2 ** 5 == 32 Bytes == 256 bits
+    hash_size_pow2 = 5
+
+    # The closest power of two bigger than the file size
+    file_size_pow2 = _next_pow2(file_size)
+
+    # Compute the target size for the 'pieces layers' in the torrent
+    # (as power of two: the closest power-of-two smaller than the number)
+    # Will cap at max_pieces_layers_size for files larger than 1TB.
+    target_pieces_layers_size = math.sqrt(file_size)
+    target_pieces_layers_size_pow2 = min(math.floor(math.log2(target_pieces_layers_size)), max_pieces_layers_size_pow2)
+    target_piece_num_pow2 = max(target_pieces_layers_size_pow2 - hash_size_pow2, 0)
+
+    piece_length_pow2 = max(file_size_pow2 - target_piece_num_pow2, min_piece_len_pow2)
+    return piece_length_pow2
+
+
+def bittorrent_v2_piece_length(file_size: int) -> int:
+    return 2 ** _bittorrent_v2_piece_length_pow2(file_size)
+
+
+def bittorrent_v2_merkle_sha256(file: "FileDescriptorOrPath") -> tuple[bytes, bytes, int]:
+    """
+    Compute the .torrent v2 hash tree for the given file.
+    (http://www.bittorrent.org/beps/bep_0052.html)
+    In particular, it will return the root of the merkle hash
+    tree of the file, the 'piece layers' as described in the
+    previous BEP, and the chosen `piece size`
+
+    This function will read the file in chunks of 16KiB
+    (which is the imposed block size by bittorrent v2) and compute
+    the sha256 hash of each block. When enough blocks are read
+    to form a `piece`, will compute the merkle hash root of the
+    piece from the hashes of its blocks. At the end, the hashes
+    of pieces are combined to create the global pieces_root.
+    """
+
+    # by the bittorrent v2 specification, the block size and the
+    # minimum piece size are both fixed to 16KiB
+    block_size = 16384
+    block_size_pow2 = 14  # 2 ** 14 == 16 KiB
+    # sha256 requires 2 ** 5 == 32 Bytes == 256 bits
+    hash_size = 32
+
+    def _merkle_root(leafs: list[bytes], nb_levels: int, padding: bytes) -> bytes:
+        """
+        Build the root of the merkle hash tree from the (possibly incomplete) leafs layer.
+        If len(leafs) < 2 ** nb_levels, it will be padded with the padding repeated as many times
+        as needed to have 2 ** nb_levels leafs in total.
+        """
+        nodes = copy.copy(leafs)
+        level = nb_levels
+
+        while level > 0:
+            for i in range(2 ** (level - 1)):
+                node1 = nodes[2 * i] if 2 * i < len(nodes) else padding
+                node2 = nodes[2 * i + 1] if 2 * i + 1 < len(nodes) else padding
+                h = hashlib.sha256(node1)
+                h.update(node2)
+                if i < len(nodes):
+                    nodes[i] = h.digest()
+                else:
+                    nodes.append(h.digest())
+            level -= 1
+        return nodes[0] if nodes else padding
+
+    file_size = os.stat(file).st_size
+    piece_length_pow2 = _bittorrent_v2_piece_length_pow2(file_size)
+
+    block_per_piece_pow2 = piece_length_pow2 - block_size_pow2
+    piece_length = 2 ** piece_length_pow2
+    block_per_piece = 2 ** block_per_piece_pow2
+    piece_num = math.ceil(file_size / piece_length)
+
+    remaining = file_size
+    remaining_in_block = min(file_size, block_size)
+    block_hashes = []
+    piece_hashes = []
+    current_hash = hashlib.sha256()
+    block_padding = bytes(hash_size)
+    with open(file, 'rb') as f:
+        while True:
+            data = f.read(remaining_in_block)
+            if not data:
+                break
+
+            current_hash.update(data)
+
+            remaining_in_block -= len(data)
+            remaining -= len(data)
+
+            if not remaining_in_block:
+                block_hashes.append(current_hash.digest())
+                if len(block_hashes) == block_per_piece or not remaining:
+                    piece_hashes.append(_merkle_root(block_hashes, nb_levels=block_per_piece_pow2, padding=block_padding))
+                    block_hashes = []
+                current_hash = hashlib.sha256()
+                remaining_in_block = min(block_size, remaining)
+
+            if not remaining:
+                break
+
+    if remaining or remaining_in_block or len(piece_hashes) != piece_num:
+        raise RucioException(f'Error while computing merkle sha256 of {file}')
+
+    piece_padding = _merkle_root([], nb_levels=block_per_piece_pow2, padding=block_padding)
+    pieces_root = _merkle_root(piece_hashes, nb_levels=_next_pow2(piece_num), padding=piece_padding)
+    pieces_layers = b''.join(piece_hashes) if len(piece_hashes) > 1 else b''
+
+    return pieces_root, pieces_layers, piece_length
+
+
+def bencode(obj: Union[int, bytes, str, list, dict[bytes, Any]]) -> bytes:
+    """
+    Copied from the reference implementation of v2 bittorrent:
+    http://bittorrent.org/beps/bep_0052_torrent_creator.py
+    """
+
+    if isinstance(obj, int):
+        return b"i" + str(obj).encode() + b"e"
+    elif isinstance(obj, bytes):
+        return str(len(obj)).encode() + b":" + obj
+    elif isinstance(obj, str):
+        return bencode(obj.encode("utf-8"))
+    elif isinstance(obj, list):
+        return b"l" + b"".join(map(bencode, obj)) + b"e"
+    elif isinstance(obj, dict):
+        if all(isinstance(i, bytes) for i in obj.keys()):
+            items = list(obj.items())
+            items.sort()
+            return b"d" + b"".join(map(bencode, itertools.chain(*items))) + b"e"
+        else:
+            raise ValueError("dict keys should be bytes " + str(obj.keys()))
+    raise ValueError("Allowed types: int, bytes, str, list, dict; not %s", type(obj))
+
+
+def construct_torrent(
+        scope: str,
+        name: str,
+        length: int,
+        piece_length: int,
+        pieces_root: bytes,
+        pieces_layers: "Optional[bytes]" = None,
+        trackers: "Optional[list[str]]" = None,
+) -> "tuple[str, bytes]":
+
+    torrent_dict = {
+        b'creation date': int(time.time()),
+        b'info': {
+            b'meta version': 2,
+            b'private': 1,
+            b'name': f'{scope}:{name}'.encode(),
+            b'piece length': piece_length,
+            b'file tree': {
+                name.encode(): {
+                    b'': {
+                        b'length': length,
+                        b'pieces root': pieces_root,
+                    }
+                }
+            }
+        },
+        b'piece layers': {},
+    }
+    if trackers:
+        torrent_dict[b'announce'] = trackers[0].encode()
+        if len(trackers) > 1:
+            torrent_dict[b'announce-list'] = [t.encode() for t in trackers]
+    if pieces_layers:
+        torrent_dict[b'piece layers'][pieces_root] = pieces_layers
+
+    torrent_id = hashlib.sha256(bencode(torrent_dict[b'info'])).hexdigest()[:40]
+    torrent = bencode(torrent_dict)
+    return torrent_id, torrent
+
+
+def merkle_sha256(file: "FileDescriptorOrPath") -> str:
+    """
+    The root of the sha256 merkle hash tree with leaf size of 16 KiB.
+    """
+    pieces_root, _, _ = bittorrent_v2_merkle_sha256(file)
+    return pieces_root.hex()

rucio/common/cache.py

@@ -12,18 +12,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Optional
 
-from dogpile.cache import make_region
+from dogpile.cache.region import CacheRegion
 
+from rucio.common.client import is_client
 from rucio.common.config import config_get
-from rucio.common.utils import is_client
 
 if TYPE_CHECKING:
     from collections.abc import Callable
-    from typing import Optional
 
-    from dogpile.cache.region import CacheRegion
 
 CACHE_URL = config_get('cache', 'url', False, '127.0.0.1:11211', check_config_table=False)
 
@@ -45,30 +43,69 @@ finally:
         _mc_client.close()
 
 
-def make_region_memcached(
-        expiration_time: int,
-        function_key_generator: "Optional[Callable]" = None,
-        memcached_expire_time: "Optional[int]" = None
-) -> "CacheRegion":
+class MemcacheRegion(CacheRegion):
     """
-    Make and configure a dogpile.cache.pymemcache region
+    Subclass of CacheRegion.
+    It uses pymemcache as backend if ENABLE_CACHING is True,
+    otherwise it it configured to null.
     """
-    if function_key_generator:
-        region = make_region(function_key_generator=function_key_generator)
-    else:
-        region = make_region()
-
-    if ENABLE_CACHING:
-        region.configure(
-            'dogpile.cache.pymemcache',
-            expiration_time=expiration_time,
-            arguments={
-                'url': CACHE_URL,
-                'distributed_lock': True,
-                'memcached_expire_time': memcached_expire_time if memcached_expire_time else expiration_time + 60,  # must be bigger than expiration_time
-            }
-        )
-    else:
-        region.configure('dogpile.cache.null')
+    def __init__(
+            self,
+            expiration_time: int,
+            function_key_generator: Optional['Callable'] = None,
+            memcached_expire_time: Optional[int] = None
+    ):
+        if function_key_generator:
+            super().__init__(function_key_generator=function_key_generator)
+        else:
+            super().__init__()
+        self._configure_region(expiration_time, memcached_expire_time)
+
+    def _configure_region(
+            self,
+            expiration_time: int,
+            memcached_expire_time: Optional[int]
+    ) -> None:
+        if ENABLE_CACHING:
+            self.configure(
+                'dogpile.cache.pymemcache',
+                expiration_time=expiration_time,
+                arguments={
+                    'url': CACHE_URL,
+                    'distributed_lock': True,
+                    'memcached_expire_time': memcached_expire_time if memcached_expire_time else expiration_time + 60,  # must be bigger than expiration_time
+                }
+            )
+        else:
+            self.configure('dogpile.cache.null')
+
+
+class CacheKey:
+    """
+    Helper class to generate cache keys
+    based on sections and options.
+    """
+
+    @staticmethod
+    def _generate_key(*args: str) -> str:
+        return '_'.join(args)
+
+    @staticmethod
+    def has_section(section: str) -> str:
+        return CacheKey._generate_key('has_section', section)
+
+    @staticmethod
+    def options(section: str) -> str:
+        return CacheKey._generate_key('options', section)
+
+    @staticmethod
+    def has_option(section: str, option: str) -> str:
+        return CacheKey._generate_key('has_option', section, option)
+
+    @staticmethod
+    def items(section: str) -> str:
+        return CacheKey._generate_key('items', section)
 
-    return region
+    @staticmethod
+    def value(section: str, option: str) -> str:
+        return CacheKey._generate_key('get', section, option)

rucio/common/checksum.py

@@ -0,0 +1,168 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import hashlib
+import io
+import mmap
+import zlib
+from functools import partial
+from typing import TYPE_CHECKING
+
+from rucio.common.bittorrent import merkle_sha256
+from rucio.common.exception import ChecksumCalculationError
+
+if TYPE_CHECKING:
+    from _typeshed import FileDescriptorOrPath
+
+# GLOBALLY_SUPPORTED_CHECKSUMS = ['adler32', 'md5', 'sha256', 'crc32']
+GLOBALLY_SUPPORTED_CHECKSUMS = ['adler32', 'md5']
+PREFERRED_CHECKSUM = GLOBALLY_SUPPORTED_CHECKSUMS[0]
+CHECKSUM_KEY = 'supported_checksums'
+
+
+def is_checksum_valid(checksum_name: str) -> bool:
+    """
+    A simple function to check whether a checksum algorithm is supported.
+    Relies on GLOBALLY_SUPPORTED_CHECKSUMS to allow for expandability.
+
+    :param checksum_name: The name of the checksum to be verified.
+    :returns: True if checksum_name is in GLOBALLY_SUPPORTED_CHECKSUMS list, False otherwise.
+    """
+
+    return checksum_name in GLOBALLY_SUPPORTED_CHECKSUMS
+
+
+def set_preferred_checksum(checksum_name: str) -> None:
+    """
+    If the input checksum name is valid,
+    set it as PREFERRED_CHECKSUM.
+
+    :param checksum_name: The name of the checksum to be verified.
+    """
+    if is_checksum_valid(checksum_name):
+        global PREFERRED_CHECKSUM
+        PREFERRED_CHECKSUM = checksum_name
+
+
+def _iter_blocks(fobj):
+    """Iterate over blocks in a binary file-like object.
+
+    Uses blocks of size ``io.DEFAULT_BUFFER_SIZE * 8``.
+    """
+    block_size = io.DEFAULT_BUFFER_SIZE * 8
+    return iter(partial(fobj.read, block_size), b'')
+
+
+def adler32(file: "FileDescriptorOrPath") -> str:
+    """
+    An Adler-32 checksum is obtained by calculating two 16-bit checksums A and B
+    and concatenating their bits into a 32-bit integer. A is the sum of all bytes in the
+    stream plus one, and B is the sum of the individual values of A from each step.
+
+    :param file: file name
+    :returns: Hexified string, padded to 8 values.
+    """
+
+    # adler starting value is _not_ 0
+    adler = 1
+
+    can_mmap = False
+    # try:
+    #    with open(file, 'r+b') as f:
+    #        can_mmap = True
+    # except:
+    #    pass
+
+    try:
+        # use mmap if possible
+        if can_mmap:
+            with open(file, 'r+b') as f:
+                m = mmap.mmap(f.fileno(), 0)
+                # partial block reads at slightly increased buffer sizes
+                for block in _iter_blocks(m):
+                    adler = zlib.adler32(block, adler)
+        else:
+            with open(file, 'rb') as f:
+                # partial block reads at slightly increased buffer sizes
+                for block in _iter_blocks(f):
+                    adler = zlib.adler32(block, adler)
+
+    except Exception as e:
+        raise ChecksumCalculationError('adler32', str(file), e)
+
+    # backflip on 32bit -- can be removed once everything is fully migrated to 64bit
+    if adler < 0:
+        adler = adler + 2 ** 32
+
+    return str('%08x' % adler)
+
+
+def md5(file: "FileDescriptorOrPath") -> str:
+    """
+    Runs the MD5 algorithm (RFC-1321) on the binary content of the file named file and returns the hexadecimal digest
+
+    :param file: file name
+    :returns: string of 32 hexadecimal digits
+    """
+    hash_md5 = hashlib.md5()
+    try:
+        with open(file, "rb") as f:
+            for block in _iter_blocks(f):
+                hash_md5.update(block)
+    except Exception as e:
+        raise ChecksumCalculationError('md5', str(file), e)
+
+    return hash_md5.hexdigest()
+
+
+def sha256(file: "FileDescriptorOrPath") -> str:
+    """
+    Runs the SHA256 algorithm on the binary content of the file named file and returns the hexadecimal digest
+
+    :param file: file name
+    :returns: string of 32 hexadecimal digits
+    """
+    checksum = hashlib.sha256()
+    try:
+        with open(file, "rb") as f:
+            for block in _iter_blocks(f):
+                checksum.update(block)
+    except Exception as e:
+        raise ChecksumCalculationError('sha256', str(file), e)
+    return checksum.hexdigest()
+
+
+def crc32(file: "FileDescriptorOrPath") -> str:
+    """
+    Runs the CRC32 algorithm on the binary content of the file named file and returns the hexadecimal digest
+
+    :param file: file name
+    :returns: string of 32 hexadecimal digits
+    """
+    prev = 0
+    try:
+        with open(file, "rb") as f:
+            for block in _iter_blocks(f):
+                prev = zlib.crc32(block, prev)
+    except Exception as e:
+        raise ChecksumCalculationError('crc32', str(file), e)
+    return "%X" % (prev & 0xFFFFFFFF)
+
+
+CHECKSUM_ALGO_DICT = {
+    'adler32': adler32,
+    'md5': md5,
+    'sha256': sha256,
+    'crc32': crc32,
+    'merkle_sha256': merkle_sha256
+}

rucio/common/client.py

@@ -0,0 +1,122 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import socket
+from configparser import NoOptionError, NoSectionError
+from typing import TYPE_CHECKING
+
+from rucio.common.config import config_get, config_has_section
+from rucio.common.exception import ConfigNotFound
+
+if TYPE_CHECKING:
+    from rucio.common.types import IPWithLocationDict
+
+
+def is_client() -> bool:
+    """"
+    Checks if the function is called from a client or from a server/daemon
+
+    :returns client_mode: True if is called from a client, False if it is called from a server/daemon
+    """
+    if 'RUCIO_CLIENT_MODE' not in os.environ:
+        try:
+            if config_has_section('database'):
+                client_mode = False
+            elif config_has_section('client'):
+                client_mode = True
+            else:
+                client_mode = False
+        except (RuntimeError, ConfigNotFound):
+            # If no configuration file is found the default value should be True
+            client_mode = True
+    else:
+        if os.environ['RUCIO_CLIENT_MODE']:
+            client_mode = True
+        else:
+            client_mode = False
+
+    return client_mode
+
+
+def get_client_vo() -> str:
+    """
+    Get the client VO from the environment or the configuration file.
+
+    :returns vo: The client VO as a string; default = 'def'.
+    """
+    if 'RUCIO_VO' in os.environ:
+        vo = os.environ['RUCIO_VO']
+    else:
+        try:
+            vo = str(config_get('client', 'vo'))
+        except (NoOptionError, NoSectionError):
+            vo = 'def'
+    return vo
+
+
+def detect_client_location() -> "IPWithLocationDict":
+    """
+    Normally client IP will be set on the server side (request.remote_addr)
+    Here setting ip on the one seen by the host itself. There is no connection
+    to Google DNS servers.
+    Try to determine the sitename automatically from common environment variables,
+    in this order: SITE_NAME, ATLAS_SITE_NAME, OSG_SITE_NAME. If none of these exist
+    use the fixed string 'ROAMING'.
+
+    If environment variables sets location, it uses it.
+    """
+
+    ip = None
+
+    try:
+        with socket.socket(socket.AF_INET6, socket.SOCK_DGRAM) as s:
+            s.connect(("2001:4860:4860:0:0:0:0:8888", 80))
+            ip = s.getsockname()[0]
+    except Exception:
+        pass
+
+    if not ip:
+        try:
+            with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as s:
+                s.connect(("8.8.8.8", 80))
+                ip = s.getsockname()[0]
+        except Exception:
+            pass
+
+    if not ip:
+        ip = '0.0.0.0'  # noqa: S104
+
+    site = os.environ.get('SITE_NAME',
+                          os.environ.get('ATLAS_SITE_NAME',
+                                         os.environ.get('OSG_SITE_NAME',
+                                                        'ROAMING')))
+
+    latitude = os.environ.get('RUCIO_LATITUDE')
+    longitude = os.environ.get('RUCIO_LONGITUDE')
+    if latitude and longitude:
+        try:
+            latitude = float(latitude)
+            longitude = float(longitude)
+        except ValueError:
+            latitude = longitude = 0
+            print('Client set latitude and longitude are not valid.')
+    else:
+        latitude = longitude = None
+
+    return {'ip': ip,
+            'fqdn': socket.getfqdn(),
+            'site': site,
+            'latitude': latitude,
+            'longitude': longitude}