rucio-clients 35.6.0__tar.gz → 36.0.0rc1__tar.gz

{rucio_clients-35.6.0 → rucio_clients-36.0.0rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: rucio-clients
-Version: 35.6.0
+Version: 36.0.0rc1
 Summary: Rucio Client Lite Package
 Home-page: https://rucio.cern.ch/
 Author: Rucio

{rucio_clients-35.6.0 → rucio_clients-36.0.0rc1}/bin/rucio

@@ -37,6 +37,7 @@ from tabulate import tabulate
 # rucio module has the same name as this executable module, so this rule fails. pylint: disable=no-name-in-module
 from rucio import version
 from rucio.client import Client
+from rucio.common.client import detect_client_location
 from rucio.common.config import config_get, config_get_float
 from rucio.common.constants import ReplicaState
 from rucio.common.exception import (
@@ -60,7 +61,7 @@ from rucio.common.exception import (
 )
 from rucio.common.extra import import_extras
 from rucio.common.test_rucio_server import TestRucioServer
-from rucio.common.utils import Color, StoreAndDeprecateWarningAction, chunks, detect_client_location, extract_scope, parse_did_filter_from_string, parse_did_filter_from_string_fe, setup_logger, sizefmt
+from rucio.common.utils import Color, StoreAndDeprecateWarningAction, chunks, extract_scope, parse_did_filter_from_string, parse_did_filter_from_string_fe, setup_logger, sizefmt
 
 EXTRA_MODULES = import_extras(['argcomplete'])
 

{rucio_clients-35.6.0 → rucio_clients-36.0.0rc1}/etc/rucio.cfg.atlas.client.template

@@ -29,8 +29,9 @@ request_retries = 3
 auth_type = x509_proxy
 
 [policy]
-permission = atlas
-schema = atlas
+package = atlas_rucio_policy_package
+#permission = atlas
+#schema = atlas
 lfn2pfn_algorithm_default = hash
 support = hn-atlas-dist-analysis-help@cern.ch
 support_rucio = https://github.com/rucio/rucio/issues/

{rucio_clients-35.6.0 → rucio_clients-36.0.0rc1}/etc/rucio.cfg.template

@@ -229,9 +229,10 @@ panda_url = http://bigpanda.cern.ch/jobs/?category=analysis&jobstatus=running
 window = 604800
 
 [policy]
-package = ATLASRucioPolicy
+package = atlas_rucio_policy_package
 permission = atlas
 schema = atlas
+extract_scope = atlas
 lfn2pfn_algorithm_default = hash
 support = hn-atlas-dist-analysis-help@cern.ch
 support_rucio = https://github.com/rucio/rucio/issues/

{rucio_clients-35.6.0 → rucio_clients-36.0.0rc1}/lib/rucio/client/baseclient.py

@@ -22,13 +22,11 @@ import os
 import secrets
 import sys
 import time
-from collections.abc import Generator
 from configparser import NoOptionError, NoSectionError
-from logging import Logger
 from os import environ, fdopen, geteuid, makedirs, path
 from shutil import move
 from tempfile import mkstemp
-from typing import Any, Optional
+from typing import TYPE_CHECKING, Any, Optional
 from urllib.parse import urlparse
 
 import requests
@@ -44,6 +42,10 @@ from rucio.common.exception import CannotAuthenticate, ClientProtocolNotSupporte
 from rucio.common.extra import import_extras
 from rucio.common.utils import build_url, get_tmp_dir, my_key_generator, parse_response, setup_logger, ssh_sign
 
+if TYPE_CHECKING:
+    from collections.abc import Generator
+    from logging import Logger
+
 EXTRA_MODULES = import_extras(['requests_kerberos'])
 
 if EXTRA_MODULES['requests_kerberos']:
@@ -90,7 +92,7 @@ class BaseClient:
                  timeout: Optional[int] = 600,
                  user_agent: Optional[str] = 'rucio-clients',
                  vo: Optional[str] = None,
-                 logger: Logger = LOG) -> None:
+                 logger: 'Logger' = LOG) -> None:
         """
         Constructor of the BaseClient.
         :param rucio_host: The address of the rucio server, if None it is read from the config file.
@@ -364,7 +366,7 @@ class BaseClient:
         else:
             return exception.RucioException, "%s: %s" % (exc_cls, exc_msg)
 
-    def _load_json_data(self, response: requests.Response) -> Generator[Any, Any, Any]:
+    def _load_json_data(self, response: requests.Response) -> 'Generator[Any, Any, Any]':
         """
         Helper method to correctly load json data based on the content type of the http response.
 

{rucio_clients-35.6.0 → rucio_clients-36.0.0rc1}/lib/rucio/client/downloadclient.py

@@ -29,11 +29,13 @@ from typing import TYPE_CHECKING, Any, Optional
 
 from rucio import version
 from rucio.client.client import Client
+from rucio.common.checksum import CHECKSUM_ALGO_DICT, GLOBALLY_SUPPORTED_CHECKSUMS, PREFERRED_CHECKSUM, adler32
+from rucio.common.client import detect_client_location
 from rucio.common.config import config_get
 from rucio.common.didtype import DID
 from rucio.common.exception import InputValidationError, NoFilesDownloaded, NotAllFilesDownloaded, RucioException
 from rucio.common.pcache import Pcache
-from rucio.common.utils import CHECKSUM_ALGO_DICT, GLOBALLY_SUPPORTED_CHECKSUMS, PREFERRED_CHECKSUM, adler32, detect_client_location, execute, extract_scope, generate_uuid, parse_replicas_from_file, parse_replicas_from_string, send_trace, sizefmt
+from rucio.common.utils import execute, extract_scope, generate_uuid, parse_replicas_from_file, parse_replicas_from_string, send_trace, sizefmt
 from rucio.rse import rsemanager as rsemgr
 
 if TYPE_CHECKING:

{rucio_clients-35.6.0 → rucio_clients-36.0.0rc1}/lib/rucio/client/rseclient.py

@@ -24,7 +24,7 @@ from rucio.common.utils import build_url
 if TYPE_CHECKING:
     from collections.abc import Iterable, Iterator
 
-    from rucio.common.constants import RSE_SUPPORTED_PROTOCOL_DOMAINS_LITERAL, RSE_SUPPORTED_PROTOCOL_OPERATIONS_LITERAL, SUPPORTED_PROTOCOLS_LITERAL
+    from rucio.common.constants import RSE_ALL_SUPPORTED_PROTOCOL_OPERATIONS_LITERAL, RSE_SUPPORTED_PROTOCOL_DOMAINS_LITERAL, SUPPORTED_PROTOCOLS_LITERAL
 
 
 class RSEClient(BaseClient):
@@ -238,7 +238,7 @@ class RSEClient(BaseClient):
             self,
             rse: str,
             protocol_domain: "RSE_SUPPORTED_PROTOCOL_DOMAINS_LITERAL" = 'ALL',
-            operation: Optional["RSE_SUPPORTED_PROTOCOL_OPERATIONS_LITERAL"] = None,
+            operation: Optional["RSE_ALL_SUPPORTED_PROTOCOL_OPERATIONS_LITERAL"] = None,
             default: bool = False,
             scheme: Optional['SUPPORTED_PROTOCOLS_LITERAL'] = None
     ) -> Any:
@@ -287,7 +287,7 @@ class RSEClient(BaseClient):
             rse: str,
             lfns: 'Iterable[str]',
             protocol_domain: 'RSE_SUPPORTED_PROTOCOL_DOMAINS_LITERAL' = 'ALL',
-            operation: Optional['RSE_SUPPORTED_PROTOCOL_OPERATIONS_LITERAL'] = None,
+            operation: Optional['RSE_ALL_SUPPORTED_PROTOCOL_OPERATIONS_LITERAL'] = None,
             scheme: Optional['SUPPORTED_PROTOCOLS_LITERAL'] = None
     ) -> dict[str, str]:
         """
@@ -409,7 +409,7 @@ class RSEClient(BaseClient):
             self,
             rse: str,
             domain: 'RSE_SUPPORTED_PROTOCOL_DOMAINS_LITERAL',
-            operation: 'RSE_SUPPORTED_PROTOCOL_OPERATIONS_LITERAL',
+            operation: 'RSE_ALL_SUPPORTED_PROTOCOL_OPERATIONS_LITERAL',
             scheme_a: 'SUPPORTED_PROTOCOLS_LITERAL',
             scheme_b: 'SUPPORTED_PROTOCOLS_LITERAL'
     ) -> bool:

{rucio_clients-35.6.0 → rucio_clients-36.0.0rc1}/lib/rucio/client/uploadclient.py

@@ -25,6 +25,9 @@ from typing import TYPE_CHECKING, Any, Final, Optional, Union, cast
 
 from rucio import version
 from rucio.client.client import Client
+from rucio.common.bittorrent import bittorrent_v2_merkle_sha256
+from rucio.common.checksum import GLOBALLY_SUPPORTED_CHECKSUMS, adler32, md5
+from rucio.common.client import detect_client_location
 from rucio.common.config import config_get, config_get_bool, config_get_int
 from rucio.common.constants import RseAttr
 from rucio.common.exception import (
@@ -42,7 +45,7 @@ from rucio.common.exception import (
     ScopeNotFound,
     ServiceUnavailable,
 )
-from rucio.common.utils import GLOBALLY_SUPPORTED_CHECKSUMS, adler32, bittorrent_v2_merkle_sha256, detect_client_location, execute, generate_uuid, make_valid_did, md5, retry, send_trace
+from rucio.common.utils import execute, generate_uuid, make_valid_did, retry, send_trace
 from rucio.rse import rsemanager as rsemgr
 
 if TYPE_CHECKING:

rucio_clients-36.0.0rc1/lib/rucio/common/bittorrent.py

@@ -0,0 +1,234 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import copy
+import hashlib
+import itertools
+import math
+import os
+import time
+from typing import TYPE_CHECKING, Any, Optional, Union
+
+from rucio.common.exception import RucioException
+
+if TYPE_CHECKING:
+    from _typeshed import FileDescriptorOrPath
+
+
+def _next_pow2(num: int) -> int:
+    if not num:
+        return 0
+    return math.ceil(math.log2(num))
+
+
+def _bittorrent_v2_piece_length_pow2(file_size: int) -> int:
+    """
+    Automatically chooses the `piece size` so that `piece layers`
+    is kept small(er) than usually. This is a balancing act:
+    having a big piece_length requires more work on bittorrent client
+    side to validate hashes, but having it small requires more
+    place to store the `piece layers` in the database.
+
+    Returns the result as the exponent 'x' for power of 2.
+    To get the actual length in bytes, the caller should compute 2^x.
+    """
+
+    # by the bittorrent v2 specification, the piece size is equal to block size = 16KiB
+    min_piece_len_pow2 = 14  # 2 ** 14 == 16 KiB
+    if not file_size:
+        return min_piece_len_pow2
+    # Limit the maximum size of pieces_layers hash chain for bittorrent v2,
+    # because we'll have to store it in the database
+    max_pieces_layers_size_pow2 = 20  # 2 ** 20 == 1 MiB
+    # sha256 requires 2 ** 5 == 32 Bytes == 256 bits
+    hash_size_pow2 = 5
+
+    # The closest power of two bigger than the file size
+    file_size_pow2 = _next_pow2(file_size)
+
+    # Compute the target size for the 'pieces layers' in the torrent
+    # (as power of two: the closest power-of-two smaller than the number)
+    # Will cap at max_pieces_layers_size for files larger than 1TB.
+    target_pieces_layers_size = math.sqrt(file_size)
+    target_pieces_layers_size_pow2 = min(math.floor(math.log2(target_pieces_layers_size)), max_pieces_layers_size_pow2)
+    target_piece_num_pow2 = max(target_pieces_layers_size_pow2 - hash_size_pow2, 0)
+
+    piece_length_pow2 = max(file_size_pow2 - target_piece_num_pow2, min_piece_len_pow2)
+    return piece_length_pow2
+
+
+def bittorrent_v2_piece_length(file_size: int) -> int:
+    return 2 ** _bittorrent_v2_piece_length_pow2(file_size)
+
+
+def bittorrent_v2_merkle_sha256(file: "FileDescriptorOrPath") -> tuple[bytes, bytes, int]:
+    """
+    Compute the .torrent v2 hash tree for the given file.
+    (http://www.bittorrent.org/beps/bep_0052.html)
+    In particular, it will return the root of the merkle hash
+    tree of the file, the 'piece layers' as described in the
+    previous BEP, and the chosen `piece size`
+
+    This function will read the file in chunks of 16KiB
+    (which is the imposed block size by bittorrent v2) and compute
+    the sha256 hash of each block. When enough blocks are read
+    to form a `piece`, will compute the merkle hash root of the
+    piece from the hashes of its blocks. At the end, the hashes
+    of pieces are combined to create the global pieces_root.
+    """
+
+    # by the bittorrent v2 specification, the block size and the
+    # minimum piece size are both fixed to 16KiB
+    block_size = 16384
+    block_size_pow2 = 14  # 2 ** 14 == 16 KiB
+    # sha256 requires 2 ** 5 == 32 Bytes == 256 bits
+    hash_size = 32
+
+    def _merkle_root(leafs: list[bytes], nb_levels: int, padding: bytes) -> bytes:
+        """
+        Build the root of the merkle hash tree from the (possibly incomplete) leafs layer.
+        If len(leafs) < 2 ** nb_levels, it will be padded with the padding repeated as many times
+        as needed to have 2 ** nb_levels leafs in total.
+        """
+        nodes = copy.copy(leafs)
+        level = nb_levels
+
+        while level > 0:
+            for i in range(2 ** (level - 1)):
+                node1 = nodes[2 * i] if 2 * i < len(nodes) else padding
+                node2 = nodes[2 * i + 1] if 2 * i + 1 < len(nodes) else padding
+                h = hashlib.sha256(node1)
+                h.update(node2)
+                if i < len(nodes):
+                    nodes[i] = h.digest()
+                else:
+                    nodes.append(h.digest())
+            level -= 1
+        return nodes[0] if nodes else padding
+
+    file_size = os.stat(file).st_size
+    piece_length_pow2 = _bittorrent_v2_piece_length_pow2(file_size)
+
+    block_per_piece_pow2 = piece_length_pow2 - block_size_pow2
+    piece_length = 2 ** piece_length_pow2
+    block_per_piece = 2 ** block_per_piece_pow2
+    piece_num = math.ceil(file_size / piece_length)
+
+    remaining = file_size
+    remaining_in_block = min(file_size, block_size)
+    block_hashes = []
+    piece_hashes = []
+    current_hash = hashlib.sha256()
+    block_padding = bytes(hash_size)
+    with open(file, 'rb') as f:
+        while True:
+            data = f.read(remaining_in_block)
+            if not data:
+                break
+
+            current_hash.update(data)
+
+            remaining_in_block -= len(data)
+            remaining -= len(data)
+
+            if not remaining_in_block:
+                block_hashes.append(current_hash.digest())
+                if len(block_hashes) == block_per_piece or not remaining:
+                    piece_hashes.append(_merkle_root(block_hashes, nb_levels=block_per_piece_pow2, padding=block_padding))
+                    block_hashes = []
+                current_hash = hashlib.sha256()
+                remaining_in_block = min(block_size, remaining)
+
+            if not remaining:
+                break
+
+    if remaining or remaining_in_block or len(piece_hashes) != piece_num:
+        raise RucioException(f'Error while computing merkle sha256 of {file}')
+
+    piece_padding = _merkle_root([], nb_levels=block_per_piece_pow2, padding=block_padding)
+    pieces_root = _merkle_root(piece_hashes, nb_levels=_next_pow2(piece_num), padding=piece_padding)
+    pieces_layers = b''.join(piece_hashes) if len(piece_hashes) > 1 else b''
+
+    return pieces_root, pieces_layers, piece_length
+
+
+def bencode(obj: Union[int, bytes, str, list, dict[bytes, Any]]) -> bytes:
+    """
+    Copied from the reference implementation of v2 bittorrent:
+    http://bittorrent.org/beps/bep_0052_torrent_creator.py
+    """
+
+    if isinstance(obj, int):
+        return b"i" + str(obj).encode() + b"e"
+    elif isinstance(obj, bytes):
+        return str(len(obj)).encode() + b":" + obj
+    elif isinstance(obj, str):
+        return bencode(obj.encode("utf-8"))
+    elif isinstance(obj, list):
+        return b"l" + b"".join(map(bencode, obj)) + b"e"
+    elif isinstance(obj, dict):
+        if all(isinstance(i, bytes) for i in obj.keys()):
+            items = list(obj.items())
+            items.sort()
+            return b"d" + b"".join(map(bencode, itertools.chain(*items))) + b"e"
+        else:
+            raise ValueError("dict keys should be bytes " + str(obj.keys()))
+    raise ValueError("Allowed types: int, bytes, str, list, dict; not %s", type(obj))
+
+
+def construct_torrent(
+        scope: str,
+        name: str,
+        length: int,
+        piece_length: int,
+        pieces_root: bytes,
+        pieces_layers: "Optional[bytes]" = None,
+        trackers: "Optional[list[str]]" = None,
+) -> "tuple[str, bytes]":
+
+    torrent_dict = {
+        b'creation date': int(time.time()),
+        b'info': {
+            b'meta version': 2,
+            b'private': 1,
+            b'name': f'{scope}:{name}'.encode(),
+            b'piece length': piece_length,
+            b'file tree': {
+                name.encode(): {
+                    b'': {
+                        b'length': length,
+                        b'pieces root': pieces_root,
+                    }
+                }
+            }
+        },
+        b'piece layers': {},
+    }
+    if trackers:
+        torrent_dict[b'announce'] = trackers[0].encode()
+        if len(trackers) > 1:
+            torrent_dict[b'announce-list'] = [t.encode() for t in trackers]
+    if pieces_layers:
+        torrent_dict[b'piece layers'][pieces_root] = pieces_layers
+
+    torrent_id = hashlib.sha256(bencode(torrent_dict[b'info'])).hexdigest()[:40]
+    torrent = bencode(torrent_dict)
+    return torrent_id, torrent
+
+
+def merkle_sha256(file: "FileDescriptorOrPath") -> str:
+    """
+    The root of the sha256 merkle hash tree with leaf size of 16 KiB.
+    """
+    pieces_root, _, _ = bittorrent_v2_merkle_sha256(file)
+    return pieces_root.hex()

rucio_clients-36.0.0rc1/lib/rucio/common/cache.py

@@ -0,0 +1,111 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import TYPE_CHECKING, Optional
+
+from dogpile.cache.region import CacheRegion
+
+from rucio.common.client import is_client
+from rucio.common.config import config_get
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+
+CACHE_URL = config_get('cache', 'url', False, '127.0.0.1:11211', check_config_table=False)
+
+ENABLE_CACHING = True
+_mc_client = None
+try:
+    if is_client():
+        ENABLE_CACHING = False
+    else:
+        import pymemcache
+        _mc_client = pymemcache.Client(CACHE_URL, connect_timeout=1, timeout=1)
+        _mc_client.version()
+except OSError:
+    ENABLE_CACHING = False
+except ImportError:
+    ENABLE_CACHING = False
+finally:
+    if _mc_client:
+        _mc_client.close()
+
+
+class MemcacheRegion(CacheRegion):
+    """
+    Subclass of CacheRegion.
+    It uses pymemcache as backend if ENABLE_CACHING is True,
+    otherwise it it configured to null.
+    """
+    def __init__(
+            self,
+            expiration_time: int,
+            function_key_generator: Optional['Callable'] = None,
+            memcached_expire_time: Optional[int] = None
+    ):
+        if function_key_generator:
+            super().__init__(function_key_generator=function_key_generator)
+        else:
+            super().__init__()
+        self._configure_region(expiration_time, memcached_expire_time)
+
+    def _configure_region(
+            self,
+            expiration_time: int,
+            memcached_expire_time: Optional[int]
+    ) -> None:
+        if ENABLE_CACHING:
+            self.configure(
+                'dogpile.cache.pymemcache',
+                expiration_time=expiration_time,
+                arguments={
+                    'url': CACHE_URL,
+                    'distributed_lock': True,
+                    'memcached_expire_time': memcached_expire_time if memcached_expire_time else expiration_time + 60,  # must be bigger than expiration_time
+                }
+            )
+        else:
+            self.configure('dogpile.cache.null')
+
+
+class CacheKey:
+    """
+    Helper class to generate cache keys
+    based on sections and options.
+    """
+
+    @staticmethod
+    def _generate_key(*args: str) -> str:
+        return '_'.join(args)
+
+    @staticmethod
+    def has_section(section: str) -> str:
+        return CacheKey._generate_key('has_section', section)
+
+    @staticmethod
+    def options(section: str) -> str:
+        return CacheKey._generate_key('options', section)
+
+    @staticmethod
+    def has_option(section: str, option: str) -> str:
+        return CacheKey._generate_key('has_option', section, option)
+
+    @staticmethod
+    def items(section: str) -> str:
+        return CacheKey._generate_key('items', section)
+
+    @staticmethod
+    def value(section: str, option: str) -> str:
+        return CacheKey._generate_key('get', section, option)

rucio_clients-36.0.0rc1/lib/rucio/common/checksum.py

@@ -0,0 +1,151 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import hashlib
+import io
+import mmap
+import zlib
+from functools import partial
+from typing import TYPE_CHECKING
+
+from rucio.common.bittorrent import merkle_sha256
+from rucio.common.exception import ChecksumCalculationError
+
+if TYPE_CHECKING:
+    from _typeshed import FileDescriptorOrPath
+
+# GLOBALLY_SUPPORTED_CHECKSUMS = ['adler32', 'md5', 'sha256', 'crc32']
+GLOBALLY_SUPPORTED_CHECKSUMS = ['adler32', 'md5']
+PREFERRED_CHECKSUM = GLOBALLY_SUPPORTED_CHECKSUMS[0]
+CHECKSUM_KEY = 'supported_checksums'
+
+
+def is_checksum_valid(checksum_name: str) -> bool:
+    """
+    A simple function to check whether a checksum algorithm is supported.
+    Relies on GLOBALLY_SUPPORTED_CHECKSUMS to allow for expandability.
+
+    :param checksum_name: The name of the checksum to be verified.
+    :returns: True if checksum_name is in GLOBALLY_SUPPORTED_CHECKSUMS list, False otherwise.
+    """
+
+    return checksum_name in GLOBALLY_SUPPORTED_CHECKSUMS
+
+
+def set_preferred_checksum(checksum_name: str) -> None:
+    """
+    If the input checksum name is valid,
+    set it as PREFERRED_CHECKSUM.
+
+    :param checksum_name: The name of the checksum to be verified.
+    """
+    if is_checksum_valid(checksum_name):
+        global PREFERRED_CHECKSUM
+        PREFERRED_CHECKSUM = checksum_name
+
+
+def adler32(file: "FileDescriptorOrPath") -> str:
+    """
+    An Adler-32 checksum is obtained by calculating two 16-bit checksums A and B
+    and concatenating their bits into a 32-bit integer. A is the sum of all bytes in the
+    stream plus one, and B is the sum of the individual values of A from each step.
+
+    :param file: file name
+    :returns: Hexified string, padded to 8 values.
+    """
+
+    # adler starting value is _not_ 0
+    adler = 1
+
+    can_mmap = False
+    # try:
+    #    with open(file, 'r+b') as f:
+    #        can_mmap = True
+    # except:
+    #    pass
+
+    try:
+        # use mmap if possible
+        if can_mmap:
+            with open(file, 'r+b') as f:
+                m = mmap.mmap(f.fileno(), 0)
+                # partial block reads at slightly increased buffer sizes
+                for block in iter(partial(m.read, io.DEFAULT_BUFFER_SIZE * 8), b''):
+                    adler = zlib.adler32(block, adler)
+        else:
+            with open(file, 'rb') as f:
+                # partial block reads at slightly increased buffer sizes
+                for block in iter(partial(f.read, io.DEFAULT_BUFFER_SIZE * 8), b''):
+                    adler = zlib.adler32(block, adler)
+
+    except Exception as e:
+        raise ChecksumCalculationError('adler32', str(file), e)
+
+    # backflip on 32bit -- can be removed once everything is fully migrated to 64bit
+    if adler < 0:
+        adler = adler + 2 ** 32
+
+    return str('%08x' % adler)
+
+
+def md5(file: "FileDescriptorOrPath") -> str:
+    """
+    Runs the MD5 algorithm (RFC-1321) on the binary content of the file named file and returns the hexadecimal digest
+
+    :param file: file name
+    :returns: string of 32 hexadecimal digits
+    """
+    hash_md5 = hashlib.md5()
+    try:
+        with open(file, "rb") as f:
+            list(map(hash_md5.update, iter(lambda: f.read(4096), b"")))
+    except Exception as e:
+        raise ChecksumCalculationError('md5', str(file), e)
+
+    return hash_md5.hexdigest()
+
+
+def sha256(file: "FileDescriptorOrPath") -> str:
+    """
+    Runs the SHA256 algorithm on the binary content of the file named file and returns the hexadecimal digest
+
+    :param file: file name
+    :returns: string of 32 hexadecimal digits
+    """
+    with open(file, "rb") as f:
+        bytes_ = f.read()  # read entire file as bytes
+        readable_hash = hashlib.sha256(bytes_).hexdigest()
+        print(readable_hash)
+        return readable_hash
+
+
+def crc32(file: "FileDescriptorOrPath") -> str:
+    """
+    Runs the CRC32 algorithm on the binary content of the file named file and returns the hexadecimal digest
+
+    :param file: file name
+    :returns: string of 32 hexadecimal digits
+    """
+    prev = 0
+    for eachLine in open(file, "rb"):
+        prev = zlib.crc32(eachLine, prev)
+    return "%X" % (prev & 0xFFFFFFFF)
+
+
+CHECKSUM_ALGO_DICT = {
+    'adler32': adler32,
+    'md5': md5,
+    'sha256': sha256,
+    'crc32': crc32,
+    'merkle_sha256': merkle_sha256
+}