rucio-clients 32.8.6__py3-none-any.whl → 35.8.0__py3-none-any.whl

rucio/common/utils.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,14 +14,18 @@
 
 import argparse
 import base64
+import copy
 import datetime
 import errno
 import getpass
 import hashlib
 import io
+import ipaddress
 import itertools
 import json
 import logging
+import math
+import mmap
 import os
 import os.path
 import re
@@ -32,26 +35,25 @@ import subprocess
 import tempfile
 import threading
 import time
+import zlib
 from collections import OrderedDict
-from configparser import NoOptionError, NoSectionError
+from collections.abc import Callable, Iterable, Iterator, Sequence
 from enum import Enum
 from functools import partial, wraps
 from io import StringIO
 from itertools import zip_longest
-from typing import TYPE_CHECKING
-from urllib.parse import urlparse, urlencode, quote, parse_qsl, urlunparse
+from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union
+from urllib.parse import parse_qsl, quote, urlencode, urlparse, urlunparse
 from uuid import uuid4 as uuid
 from xml.etree import ElementTree
 
-import mmap
 import requests
-import zlib
 
 from rucio.common.config import config_get, config_has_section
-from rucio.common.exception import MissingModuleException, InvalidType, InputValidationError, MetalinkJsonParsingError, RucioException, \
-    DuplicateCriteriaInDIDFilter, DIDFilterSyntaxError, InvalidAlgorithmName, PolicyPackageVersionError
+from rucio.common.exception import ConfigNotFound, DIDFilterSyntaxError, DuplicateCriteriaInDIDFilter, InputValidationError, InvalidType, MetalinkJsonParsingError, MissingModuleException, PolicyPackageVersionError, RucioException
 from rucio.common.extra import import_extras
-from rucio.common.types import InternalAccount, InternalScope
+from rucio.common.plugins import PolicyPackageAlgorithms
+from rucio.common.types import InternalAccount, InternalScope, TraceDict
 
 EXTRA_MODULES = import_extras(['paramiko'])
 
@@ -62,10 +64,11 @@ if EXTRA_MODULES['paramiko']:
         EXTRA_MODULES['paramiko'] = False
 
 if TYPE_CHECKING:
-    from collections.abc import Callable
-    from typing import TypeVar
-
     T = TypeVar('T')
+    from _typeshed import FileDescriptorOrPath
+    from sqlalchemy.orm import Session
+
+    from rucio.common.types import IPDict, LoggerFunction
 
 
 # HTTP code dictionary. Not complete. Can be extended if needed.
@@ -98,7 +101,7 @@ codes = {
 DATE_FORMAT = '%a, %d %b %Y %H:%M:%S UTC'
 
 
-def invert_dict(d):
+def invert_dict(d: dict[Any, Any]) -> dict[Any, Any]:
     """
     Invert the dictionary.
     CAUTION: this function is not deterministic unless the input dictionary is one-to-one mapping.
@@ -109,11 +112,11 @@ def invert_dict(d):
     return {value: key for key, value in d.items()}
 
 
-def dids_as_dicts(did_list):
+def dids_as_dicts(did_list: Iterable[Union[str, dict[str, str]]]) -> list[dict[str, str]]:
     """
     Converts list of DIDs to list of dictionaries
-    :param did_list: list of DIDs as either "scope:name" or {"scope":"scope", "name","name"}
-    :returns: list of dictionaries {"scope":"scope", "name","name"}
+    :param did_list: list of DIDs as either "scope:name" or {"scope":"scope", "name":"name"}
+    :returns: list of dictionaries {"scope":"scope", "name":"name"}
     """
     out = []
     for did in did_list:
@@ -129,7 +132,12 @@ def dids_as_dicts(did_list):
     return out
 
 
-def build_url(url, path=None, params=None, doseq=False):
+def build_url(
+        url: str,
+        path: Optional[str] = None,
+        params: Optional[Union[str, dict[Any, Any], list[tuple[Any, Any]]]] = None,
+        doseq: bool = False
+) -> str:
     """
     utitily function to build an url for requests to the rucio system.
 
@@ -148,7 +156,13 @@ def build_url(url, path=None, params=None, doseq=False):
     return complete_url
 
 
-def all_oidc_req_claims_present(scope, audience, required_scope, required_audience, sepatator=" "):
+def all_oidc_req_claims_present(
+        scope: Optional[Union[str, list[str]]],
+        audience: Optional[Union[str, list[str]]],
+        required_scope: Optional[Union[str, list[str]]],
+        required_audience: Optional[Union[str, list[str]]],
+        separator: str = " "
+) -> bool:
     """
     Checks if both of the following statements are true:
     - all items in required_scope are present in scope string
@@ -160,7 +174,7 @@ def all_oidc_req_claims_present(scope, audience, required_scope, required_audien
     :params audience: list of strings or one string where items are separated by a separator input variable
     :params required_scope: list of strings or one string where items are separated by a separator input variable
     :params required_audience: list of strings or one string where items are separated by a separator input variable
-    :params sepatator: separator string, space by default
+    :params separator: separator string, space by default
     :returns : True or False
     """
     if not scope:
@@ -184,34 +198,34 @@ def all_oidc_req_claims_present(scope, audience, required_scope, required_audien
         audience = str(audience)
         required_scope = str(required_scope)
         required_audience = str(required_audience)
-        req_scope_present = all(elem in scope.split(sepatator) for elem in required_scope.split(sepatator))
-        req_audience_present = all(elem in audience.split(sepatator) for elem in required_audience.split(sepatator))
+        req_scope_present = all(elem in scope.split(separator) for elem in required_scope.split(separator))
+        req_audience_present = all(elem in audience.split(separator) for elem in required_audience.split(separator))
         return req_scope_present and req_audience_present
     elif (isinstance(scope, list) and isinstance(audience, list) and isinstance(required_scope, str) and isinstance(required_audience, str)):
         scope = [str(it) for it in scope]
         audience = [str(it) for it in audience]
         required_scope = str(required_scope)
         required_audience = str(required_audience)
-        req_scope_present = all(elem in scope for elem in required_scope.split(sepatator))
-        req_audience_present = all(elem in audience for elem in required_audience.split(sepatator))
+        req_scope_present = all(elem in scope for elem in required_scope.split(separator))
+        req_audience_present = all(elem in audience for elem in required_audience.split(separator))
         return req_scope_present and req_audience_present
     elif (isinstance(scope, str) and isinstance(audience, str) and isinstance(required_scope, list) and isinstance(required_audience, list)):
         scope = str(scope)
         audience = str(audience)
         required_scope = [str(it) for it in required_scope]
         required_audience = [str(it) for it in required_audience]
-        req_scope_present = all(elem in scope.split(sepatator) for elem in required_scope)
-        req_audience_present = all(elem in audience.split(sepatator) for elem in required_audience)
+        req_scope_present = all(elem in scope.split(separator) for elem in required_scope)
+        req_audience_present = all(elem in audience.split(separator) for elem in required_audience)
         return req_scope_present and req_audience_present
     else:
         return False
 
 
-def generate_uuid():
+def generate_uuid() -> str:
     return str(uuid()).replace('-', '').lower()
 
 
-def generate_uuid_bytes():
+def generate_uuid_bytes() -> bytes:
     return uuid().bytes
 
 
@@ -222,9 +236,9 @@ PREFERRED_CHECKSUM = GLOBALLY_SUPPORTED_CHECKSUMS[0]
 CHECKSUM_KEY = 'supported_checksums'
 
 
-def is_checksum_valid(checksum_name):
+def is_checksum_valid(checksum_name: str) -> bool:
     """
-    A simple function to check wether a checksum algorithm is supported.
+    A simple function to check whether a checksum algorithm is supported.
     Relies on GLOBALLY_SUPPORTED_CHECKSUMS to allow for expandability.
 
     :param checksum_name: The name of the checksum to be verified.
@@ -234,20 +248,19 @@ def is_checksum_valid(checksum_name):
     return checksum_name in GLOBALLY_SUPPORTED_CHECKSUMS
 
 
-def set_preferred_checksum(checksum_name):
+def set_preferred_checksum(checksum_name: str) -> None:
     """
-    A simple function to check wether a checksum algorithm is supported.
-    Relies on GLOBALLY_SUPPORTED_CHECKSUMS to allow for expandability.
+    If the input checksum name is valid,
+    set it as PREFERRED_CHECKSUM.
 
     :param checksum_name: The name of the checksum to be verified.
-    :returns: True if checksum_name is in GLOBALLY_SUPPORTED_CHECKSUMS list, False otherwise.
     """
     if is_checksum_valid(checksum_name):
         global PREFERRED_CHECKSUM
         PREFERRED_CHECKSUM = checksum_name
 
 
-def adler32(file):
+def adler32(file: "FileDescriptorOrPath") -> str:
     """
     An Adler-32 checksum is obtained by calculating two 16-bit checksums A and B
     and concatenating their bits into a 32-bit integer. A is the sum of all bytes in the
@@ -294,7 +307,7 @@ def adler32(file):
 CHECKSUM_ALGO_DICT['adler32'] = adler32
 
 
-def md5(file):
+def md5(file: "FileDescriptorOrPath") -> str:
     """
     Runs the MD5 algorithm (RFC-1321) on the binary content of the file named file and returns the hexadecimal digest
 
@@ -314,7 +327,7 @@ def md5(file):
 CHECKSUM_ALGO_DICT['md5'] = md5
 
 
-def sha256(file):
+def sha256(file: "FileDescriptorOrPath") -> str:
     """
     Runs the SHA256 algorithm on the binary content of the file named file and returns the hexadecimal digest
 
@@ -331,7 +344,7 @@ def sha256(file):
 CHECKSUM_ALGO_DICT['sha256'] = sha256
 
 
-def crc32(file):
+def crc32(file: "FileDescriptorOrPath") -> str:
     """
     Runs the CRC32 algorithm on the binary content of the file named file and returns the hexadecimal digest
 
@@ -347,7 +360,219 @@ def crc32(file):
 CHECKSUM_ALGO_DICT['crc32'] = crc32
 
 
-def str_to_date(string):
+def _next_pow2(num: int) -> int:
+    if not num:
+        return 0
+    return math.ceil(math.log2(num))
+
+
+def _bittorrent_v2_piece_length_pow2(file_size: int) -> int:
+    """
+    Automatically chooses the `piece size` so that `piece layers`
+    is kept small(er) than usually. This is a balancing act:
+    having a big piece_length requires more work on bittorrent client
+    side to validate hashes, but having it small requires more
+    place to store the `piece layers` in the database.
+
+    Returns the result as the exponent 'x' for power of 2.
+    To get the actual length in bytes, the caller should compute 2^x.
+    """
+
+    # by the bittorrent v2 specification, the piece size is equal to block size = 16KiB
+    min_piece_len_pow2 = 14  # 2 ** 14 == 16 KiB
+    if not file_size:
+        return min_piece_len_pow2
+    # Limit the maximum size of pieces_layers hash chain for bittorrent v2,
+    # because we'll have to store it in the database
+    max_pieces_layers_size_pow2 = 20  # 2 ** 20 == 1 MiB
+    # sha256 requires 2 ** 5 == 32 Bytes == 256 bits
+    hash_size_pow2 = 5
+
+    # The closest power of two bigger than the file size
+    file_size_pow2 = _next_pow2(file_size)
+
+    # Compute the target size for the 'pieces layers' in the torrent
+    # (as power of two: the closest power-of-two smaller than the number)
+    # Will cap at max_pieces_layers_size for files larger than 1TB.
+    target_pieces_layers_size = math.sqrt(file_size)
+    target_pieces_layers_size_pow2 = min(math.floor(math.log2(target_pieces_layers_size)), max_pieces_layers_size_pow2)
+    target_piece_num_pow2 = max(target_pieces_layers_size_pow2 - hash_size_pow2, 0)
+
+    piece_length_pow2 = max(file_size_pow2 - target_piece_num_pow2, min_piece_len_pow2)
+    return piece_length_pow2
+
+
+def bittorrent_v2_piece_length(file_size: int) -> int:
+    return 2 ** _bittorrent_v2_piece_length_pow2(file_size)
+
+
+def bittorrent_v2_merkle_sha256(file: "FileDescriptorOrPath") -> tuple[bytes, bytes, int]:
+    """
+    Compute the .torrent v2 hash tree for the given file.
+    (http://www.bittorrent.org/beps/bep_0052.html)
+    In particular, it will return the root of the merkle hash
+    tree of the file, the 'piece layers' as described in the
+    previous BEP, and the chosen `piece size`
+
+    This function will read the file in chunks of 16KiB
+    (which is the imposed block size by bittorrent v2) and compute
+    the sha256 hash of each block. When enough blocks are read
+    to form a `piece`, will compute the merkle hash root of the
+    piece from the hashes of its blocks. At the end, the hashes
+    of pieces are combined to create the global pieces_root.
+    """
+
+    # by the bittorrent v2 specification, the block size and the
+    # minimum piece size are both fixed to 16KiB
+    block_size = 16384
+    block_size_pow2 = 14  # 2 ** 14 == 16 KiB
+    # sha256 requires 2 ** 5 == 32 Bytes == 256 bits
+    hash_size = 32
+
+    def _merkle_root(leafs: list[bytes], nb_levels: int, padding: bytes) -> bytes:
+        """
+        Build the root of the merkle hash tree from the (possibly incomplete) leafs layer.
+        If len(leafs) < 2 ** nb_levels, it will be padded with the padding repeated as many times
+        as needed to have 2 ** nb_levels leafs in total.
+        """
+        nodes = copy.copy(leafs)
+        level = nb_levels
+
+        while level > 0:
+            for i in range(2 ** (level - 1)):
+                node1 = nodes[2 * i] if 2 * i < len(nodes) else padding
+                node2 = nodes[2 * i + 1] if 2 * i + 1 < len(nodes) else padding
+                h = hashlib.sha256(node1)
+                h.update(node2)
+                if i < len(nodes):
+                    nodes[i] = h.digest()
+                else:
+                    nodes.append(h.digest())
+            level -= 1
+        return nodes[0] if nodes else padding
+
+    file_size = os.stat(file).st_size
+    piece_length_pow2 = _bittorrent_v2_piece_length_pow2(file_size)
+
+    block_per_piece_pow2 = piece_length_pow2 - block_size_pow2
+    piece_length = 2 ** piece_length_pow2
+    block_per_piece = 2 ** block_per_piece_pow2
+    piece_num = math.ceil(file_size / piece_length)
+
+    remaining = file_size
+    remaining_in_block = min(file_size, block_size)
+    block_hashes = []
+    piece_hashes = []
+    current_hash = hashlib.sha256()
+    block_padding = bytes(hash_size)
+    with open(file, 'rb') as f:
+        while True:
+            data = f.read(remaining_in_block)
+            if not data:
+                break
+
+            current_hash.update(data)
+
+            remaining_in_block -= len(data)
+            remaining -= len(data)
+
+            if not remaining_in_block:
+                block_hashes.append(current_hash.digest())
+                if len(block_hashes) == block_per_piece or not remaining:
+                    piece_hashes.append(_merkle_root(block_hashes, nb_levels=block_per_piece_pow2, padding=block_padding))
+                    block_hashes = []
+                current_hash = hashlib.sha256()
+                remaining_in_block = min(block_size, remaining)
+
+            if not remaining:
+                break
+
+    if remaining or remaining_in_block or len(piece_hashes) != piece_num:
+        raise RucioException(f'Error while computing merkle sha256 of {file}')
+
+    piece_padding = _merkle_root([], nb_levels=block_per_piece_pow2, padding=block_padding)
+    pieces_root = _merkle_root(piece_hashes, nb_levels=_next_pow2(piece_num), padding=piece_padding)
+    pieces_layers = b''.join(piece_hashes) if len(piece_hashes) > 1 else b''
+
+    return pieces_root, pieces_layers, piece_length
+
+
+def merkle_sha256(file: "FileDescriptorOrPath") -> str:
+    """
+    The root of the sha256 merkle hash tree with leaf size of 16 KiB.
+    """
+    pieces_root, _, _ = bittorrent_v2_merkle_sha256(file)
+    return pieces_root.hex()
+
+
+CHECKSUM_ALGO_DICT['merkle_sha256'] = merkle_sha256
+
+
+def bencode(obj: Union[int, bytes, str, list, dict[bytes, Any]]) -> bytes:
+    """
+    Copied from the reference implementation of v2 bittorrent:
+    http://bittorrent.org/beps/bep_0052_torrent_creator.py
+    """
+
+    if isinstance(obj, int):
+        return b"i" + str(obj).encode() + b"e"
+    elif isinstance(obj, bytes):
+        return str(len(obj)).encode() + b":" + obj
+    elif isinstance(obj, str):
+        return bencode(obj.encode("utf-8"))
+    elif isinstance(obj, list):
+        return b"l" + b"".join(map(bencode, obj)) + b"e"
+    elif isinstance(obj, dict):
+        if all(isinstance(i, bytes) for i in obj.keys()):
+            items = list(obj.items())
+            items.sort()
+            return b"d" + b"".join(map(bencode, itertools.chain(*items))) + b"e"
+        else:
+            raise ValueError("dict keys should be bytes " + str(obj.keys()))
+    raise ValueError("Allowed types: int, bytes, str, list, dict; not %s", type(obj))
+
+
+def construct_torrent(
+        scope: str,
+        name: str,
+        length: int,
+        piece_length: int,
+        pieces_root: bytes,
+        pieces_layers: "Optional[bytes]" = None,
+        trackers: "Optional[list[str]]" = None,
+) -> "tuple[str, bytes]":
+
+    torrent_dict = {
+        b'creation date': int(time.time()),
+        b'info': {
+            b'meta version': 2,
+            b'private': 1,
+            b'name': f'{scope}:{name}'.encode(),
+            b'piece length': piece_length,
+            b'file tree': {
+                name.encode(): {
+                    b'': {
+                        b'length': length,
+                        b'pieces root': pieces_root,
+                    }
+                }
+            }
+        },
+        b'piece layers': {},
+    }
+    if trackers:
+        torrent_dict[b'announce'] = trackers[0].encode()
+        if len(trackers) > 1:
+            torrent_dict[b'announce-list'] = [t.encode() for t in trackers]
+    if pieces_layers:
+        torrent_dict[b'piece layers'][pieces_root] = pieces_layers
+
+    torrent_id = hashlib.sha256(bencode(torrent_dict[b'info'])).hexdigest()[:40]
+    torrent = bencode(torrent_dict)
+    return torrent_id, torrent
+
+
+def str_to_date(string: str) -> Optional[datetime.datetime]:
     """ Converts a RFC-1123 string to the corresponding datetime value.
 
     :param string: the RFC-1123 string to convert to datetime value.
@@ -355,7 +580,7 @@ def str_to_date(string):
     return datetime.datetime.strptime(string, DATE_FORMAT) if string else None
 
 
-def val_to_space_sep_str(vallist):
+def val_to_space_sep_str(vallist: list[str]) -> str:
     """ Converts a list of values into a string of space separated values
 
     :param vallist: the list of values to to convert into string
@@ -367,10 +592,10 @@ def val_to_space_sep_str(vallist):
         else:
             return str(vallist)
     except:
-        return str('')
+        return ''
 
 
-def date_to_str(date):
+def date_to_str(date: datetime.datetime) -> Optional[str]:
     """ Converts a datetime value to the corresponding RFC-1123 string.
 
     :param date: the datetime value to convert.
@@ -400,19 +625,18 @@ class APIEncoder(json.JSONEncoder):
         return json.JSONEncoder.default(self, obj)
 
 
-def render_json(**data):
-    """ JSON render function
-    """
+def render_json(*args, **kwargs) -> str:
+    """ Render a list or a dict as a JSON-formatted string. """
+    if args and isinstance(args[0], list):
+        data = args[0]
+    elif isinstance(kwargs, dict):
+        data = kwargs
+    else:
+        raise ValueError("Error while serializing object to JSON-formatted string: supported input types are list or dict.")
     return json.dumps(data, cls=APIEncoder)
 
 
-def render_json_list(list_):
-    """ JSON render function for list
-    """
-    return json.dumps(list_, cls=APIEncoder)
-
-
-def datetime_parser(dct):
+def datetime_parser(dct: dict[Any, Any]) -> dict[Any, Any]:
     """ datetime parser
     """
     for k, v in list(dct.items()):
@@ -424,17 +648,17 @@ def datetime_parser(dct):
     return dct
 
 
-def parse_response(data):
+def parse_response(data: Union[str, bytes, bytearray]) -> Any:
     """
     JSON render function
     """
-    if hasattr(data, 'decode'):
+    if isinstance(data, (bytes, bytearray)):
         data = data.decode('utf-8')
 
     return json.loads(data, object_hook=datetime_parser)
 
 
-def execute(cmd) -> tuple[int, str, str]:
+def execute(cmd: str) -> tuple[int, str, str]:
     """
     Executes a command in a subprocess. Returns a tuple
     of (exitcode, out, err), where out is the string output
@@ -456,17 +680,12 @@ def execute(cmd) -> tuple[int, str, str]:
     return exitcode, out.decode(encoding='utf-8'), err.decode(encoding='utf-8')
 
 
-def rse_supported_protocol_operations():
-    """ Returns a list with operations supported by all RSE protocols."""
-    return ['read', 'write', 'delete', 'third_party_copy_read', 'third_party_copy_write']
-
-
-def rse_supported_protocol_domains():
-    """ Returns a list with all supoorted RSE protocol domains."""
+def rse_supported_protocol_domains() -> list[str]:
+    """ Returns a list with all supported RSE protocol domains."""
     return ['lan', 'wan']
 
 
-def grouper(iterable, n, fillvalue=None):
+def grouper(iterable: Iterable[Any], n: int, fillvalue: Optional[object] = None) -> zip_longest:
     """ Collect data into fixed-length chunks or blocks """
     # grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx
     args = [iter(iterable)] * n
@@ -489,7 +708,7 @@ def chunks(iterable, n):
             yield chunk
 
 
-def dict_chunks(dict_, n):
+def dict_chunks(dict_: dict[Any, Any], n: int) -> Iterator[dict[Any, Any]]:
     """
     Iterate over the dictionary in groups of the requested size
     """
@@ -498,314 +717,394 @@ def dict_chunks(dict_, n):
         yield {k: dict_[k] for k in itertools.islice(it, n)}
 
 
-def my_key_generator(namespace, fn, **kw):
+def my_key_generator(namespace: str, fn: Callable, **kw) -> Callable[..., str]:
     """
-    Customyzed key generator for dogpile
+    Customized key generator for dogpile
     """
     fname = fn.__name__
 
-    def generate_key(*arg, **kw):
+    def generate_key(*arg, **kw) -> str:
         return namespace + "_" + fname + "_".join(str(s) for s in filter(None, arg))
 
     return generate_key
 
 
-def construct_surl_DQ2(dsn: str, scope: str, filename: str) -> str:
-    """
-    Defines relative SURL for new replicas. This method
-    contains DQ2 convention. To be used for non-deterministic sites.
-    Method imported from DQ2.
-
-    @return: relative SURL for new replica.
-    @rtype: str
-    """
-    # check how many dots in dsn
-    fields = dsn.split('.')
-    nfields = len(fields)
-
-    if nfields == 0:
-        return '/other/other/%s' % (filename)
-    elif nfields == 1:
-        stripped_dsn = __strip_dsn(dsn)
-        return '/other/%s/%s' % (stripped_dsn, filename)
-    elif nfields == 2:
-        project = fields[0]
-        stripped_dsn = __strip_dsn(dsn)
-        return '/%s/%s/%s' % (project, stripped_dsn, filename)
-    elif nfields < 5 or re.match('user*|group*', fields[0]):
-        project = fields[0]
-        f2 = fields[1]
-        f3 = fields[2]
-        stripped_dsn = __strip_dsn(dsn)
-        return '/%s/%s/%s/%s/%s' % (project, f2, f3, stripped_dsn, filename)
-    else:
-        project = fields[0]
-        dataset_type = fields[4]
-        if nfields == 5:
-            tag = 'other'
-        else:
-            tag = __strip_tag(fields[-1])
-        stripped_dsn = __strip_dsn(dsn)
-        return '/%s/%s/%s/%s/%s' % (project, dataset_type, tag, stripped_dsn, filename)
+NonDeterministicPFNAlgorithmsT = TypeVar('NonDeterministicPFNAlgorithmsT', bound='NonDeterministicPFNAlgorithms')
 
 
-def construct_surl_T0(dsn: str, scope: str, filename: str) -> str:
+class NonDeterministicPFNAlgorithms(PolicyPackageAlgorithms):
     """
-    Defines relative SURL for new replicas. This method
-    contains Tier0 convention. To be used for non-deterministic sites.
-
-    @return: relative SURL for new replica.
-    @rtype: str
+    Handle PFN construction for non-deterministic RSEs, including registration of algorithms
+    from policy packages
     """
-    fields = dsn.split('.')
-    nfields = len(fields)
-    if nfields >= 3:
-        return '/%s/%s/%s/%s/%s' % (fields[0], fields[2], fields[1], dsn, filename)
-    elif nfields == 1:
-        return '/%s/%s/%s/%s/%s' % (fields[0], 'other', 'other', dsn, filename)
-    elif nfields == 2:
-        return '/%s/%s/%s/%s/%s' % (fields[0], fields[2], 'other', dsn, filename)
-    elif nfields == 0:
-        return '/other/other/other/other/%s' % (filename)
 
+    _algorithm_type = 'non_deterministic_pfn'
 
-def construct_surl_BelleII(dsn: str, scope: str, filename: str) -> str:
-    """
-    Defines relative SURL for Belle II specific replicas.
-    This method contains the Belle II convention.
-    To be used for non-deterministic Belle II sites.
-    DSN (or datablock in the Belle II naming) contains /
+    def __init__(self) -> None:
+        """
+        Initialises a non-deterministic PFN construction object
+        """
+        super().__init__()
 
-    """
+    def construct_non_deterministic_pfn(self, dsn: str, scope: Optional[str], filename: str, naming_convention: str) -> str:
+        """
+        Calls the correct algorithm to generate a non-deterministic PFN
+        """
+        return self.get_algorithm(naming_convention)(dsn, scope, filename)
 
-    fields = dsn.split("/")
-    nfields = len(fields)
-    if nfields == 0:
-        return '/other/%s' % (filename)
-    else:
-        return '%s/%s' % (dsn, filename)
+    @classmethod
+    def supports(cls: type[NonDeterministicPFNAlgorithmsT], naming_convention: str) -> bool:
+        """
+        Checks whether a non-deterministic PFN algorithm is supported
+        """
+        return super()._supports(cls._algorithm_type, naming_convention)
 
+    @classmethod
+    def _module_init_(cls: type[NonDeterministicPFNAlgorithmsT]) -> None:
+        """
+        Registers the included non-deterministic PFN algorithms
+        """
+        cls.register('T0', cls.construct_non_deterministic_pfn_T0)
+        cls.register('DQ2', cls.construct_non_deterministic_pfn_DQ2)
+        cls.register('BelleII', cls.construct_non_deterministic_pfn_BelleII)
 
-_SURL_ALGORITHMS = {}
-_DEFAULT_SURL = 'DQ2'
-_loaded_policy_modules = False
+    @classmethod
+    def get_algorithm(cls: type[NonDeterministicPFNAlgorithmsT], naming_convention: str) -> Callable[[str, Optional[str], str], str]:
+        """
+        Looks up a non-deterministic PFN algorithm by name
+        """
+        return super()._get_one_algorithm(cls._algorithm_type, naming_convention)
 
+    @classmethod
+    def register(cls: type[NonDeterministicPFNAlgorithmsT], name: str, fn_construct_non_deterministic_pfn: Callable[[str, Optional[str], str], Optional[str]]) -> None:
+        """
+        Register a new non-deterministic PFN algorithm
+        """
+        algorithm_dict = {name: fn_construct_non_deterministic_pfn}
+        super()._register(cls._algorithm_type, algorithm_dict)
 
-def register_surl_algorithm(surl_callable, name=None):
-    if name is None:
-        name = surl_callable.__name__
-    _SURL_ALGORITHMS[name] = surl_callable
+    @staticmethod
+    def __strip_dsn(dsn: str) -> str:
+        """
+        Drop the _sub and _dis suffixes for panda datasets from the lfc path
+        they will be registered in.
+        Method imported from DQ2.
+        """
 
+        suffixes_to_drop = ['_dis', '_sub', '_frag']
+        fields = dsn.split('.')
+        last_field = fields[-1]
+        try:
+            for suffix in suffixes_to_drop:
+                last_field = re.sub('%s.*$' % suffix, '', last_field)
+        except IndexError:
+            return dsn
+        fields[-1] = last_field
+        stripped_dsn = '.'.join(fields)
+        return stripped_dsn
+
+    @staticmethod
+    def __strip_tag(tag: str) -> str:
+        """
+        Drop the _sub and _dis suffixes for panda datasets from the lfc path
+        they will be registered in
+        Method imported from DQ2.
+        """
+        suffixes_to_drop = ['_dis', '_sub', '_tid']
+        stripped_tag = tag
+        try:
+            for suffix in suffixes_to_drop:
+                stripped_tag = re.sub('%s.*$' % suffix, '', stripped_tag)
+        except IndexError:
+            return stripped_tag
+        return stripped_tag
 
-register_surl_algorithm(construct_surl_T0, 'T0')
-register_surl_algorithm(construct_surl_DQ2, 'DQ2')
-register_surl_algorithm(construct_surl_BelleII, 'BelleII')
+    @staticmethod
+    def construct_non_deterministic_pfn_DQ2(dsn: str, scope: Optional[str], filename: str) -> str:
+        """
+        Defines relative PFN for new replicas. This method
+        contains DQ2 convention. To be used for non-deterministic sites.
+        Method imported from DQ2.
+
+        @return: relative PFN for new replica.
+        @rtype: str
+        """
+        # check how many dots in dsn
+        fields = dsn.split('.')
+        nfields = len(fields)
+
+        if nfields == 0:
+            return '/other/other/%s' % (filename)
+        elif nfields == 1:
+            stripped_dsn = NonDeterministicPFNAlgorithms.__strip_dsn(dsn)
+            return '/other/%s/%s' % (stripped_dsn, filename)
+        elif nfields == 2:
+            project = fields[0]
+            stripped_dsn = NonDeterministicPFNAlgorithms.__strip_dsn(dsn)
+            return '/%s/%s/%s' % (project, stripped_dsn, filename)
+        elif nfields < 5 or re.match('user*|group*', fields[0]):
+            project = fields[0]
+            f2 = fields[1]
+            f3 = fields[2]
+            stripped_dsn = NonDeterministicPFNAlgorithms.__strip_dsn(dsn)
+            return '/%s/%s/%s/%s/%s' % (project, f2, f3, stripped_dsn, filename)
+        else:
+            project = fields[0]
+            dataset_type = fields[4]
+            if nfields == 5:
+                tag = 'other'
+            else:
+                tag = NonDeterministicPFNAlgorithms.__strip_tag(fields[-1])
+            stripped_dsn = NonDeterministicPFNAlgorithms.__strip_dsn(dsn)
+            return '/%s/%s/%s/%s/%s' % (project, dataset_type, tag, stripped_dsn, filename)
+
+    @staticmethod
+    def construct_non_deterministic_pfn_T0(dsn: str, scope: Optional[str], filename: str) -> Optional[str]:
+        """
+        Defines relative PFN for new replicas. This method
+        contains Tier0 convention. To be used for non-deterministic sites.
+
+        @return: relative PFN for new replica.
+        @rtype: str
+        """
+        fields = dsn.split('.')
+        nfields = len(fields)
+        if nfields >= 3:
+            return '/%s/%s/%s/%s/%s' % (fields[0], fields[2], fields[1], dsn, filename)
+        elif nfields == 1:
+            return '/%s/%s/%s/%s/%s' % (fields[0], 'other', 'other', dsn, filename)
+        elif nfields == 2:
+            return '/%s/%s/%s/%s/%s' % (fields[0], fields[2], 'other', dsn, filename)
+        elif nfields == 0:
+            return '/other/other/other/other/%s' % (filename)
+
+    @staticmethod
+    def construct_non_deterministic_pfn_BelleII(dsn: str, scope: Optional[str], filename: str) -> str:
+        """
+        Defines relative PFN for Belle II specific replicas.
+        This method contains the Belle II convention.
+        To be used for non-deterministic Belle II sites.
+        DSN (or datablock in the Belle II naming) contains /
+        """
+
+        fields = dsn.split("/")
+        nfields = len(fields)
+        if nfields == 0:
+            return '/other/%s' % (filename)
+        else:
+            return '%s/%s' % (dsn, filename)
 
 
-def construct_surl(dsn: str, scope: str, filename: str, naming_convention: str = None) -> str:
+_DEFAULT_NON_DETERMINISTIC_PFN = 'DQ2'
+NonDeterministicPFNAlgorithms._module_init_()
+
+
+def construct_non_deterministic_pfn(dsn: str, scope: Optional[str], filename: str, naming_convention: Optional[str] = None) -> str:
     """
-    Applies non-deterministic source url convention to the given replica.
+    Applies non-deterministic PFN convention to the given replica.
     use the naming_convention to call the actual function which will do the job.
-    Rucio administrators can potentially register additional surl generation algorithms,
+    Rucio administrators can potentially register additional PFN generation algorithms,
     which are not implemented inside this main rucio repository, so changing the
     argument list must be done with caution.
     """
-    global _loaded_policy_modules
-    if not _loaded_policy_modules:
-        # on first call, register any SURL functions from the policy packages
-        register_policy_package_algorithms('surl', _SURL_ALGORITHMS)
-        _loaded_policy_modules = True
+    pfn_algorithms = NonDeterministicPFNAlgorithms()
+    if naming_convention is None or not NonDeterministicPFNAlgorithms.supports(naming_convention):
+        naming_convention = _DEFAULT_NON_DETERMINISTIC_PFN
+    return pfn_algorithms.construct_non_deterministic_pfn(dsn, scope, filename, naming_convention)
 
-    if naming_convention is None or naming_convention not in _SURL_ALGORITHMS:
-        naming_convention = _DEFAULT_SURL
-    return _SURL_ALGORITHMS[naming_convention](dsn, scope, filename)
 
+def clean_pfns(pfns: Iterable[str]) -> list[str]:
+    res = []
+    for pfn in pfns:
+        if pfn.startswith('srm'):
+            pfn = re.sub(':[0-9]+/', '/', pfn)
+            pfn = re.sub(r'/srm/managerv1\?SFN=', '', pfn)
+            pfn = re.sub(r'/srm/v2/server\?SFN=', '', pfn)
+            pfn = re.sub(r'/srm/managerv2\?SFN=', '', pfn)
+        if '?GoogleAccessId' in pfn:
+            pfn = pfn.split('?GoogleAccessId')[0]
+        if '?X-Amz' in pfn:
+            pfn = pfn.split('?X-Amz')[0]
+        res.append(pfn)
+    res.sort()
+    return res
 
-def __strip_dsn(dsn):
-    """
-    Drop the _sub and _dis suffixes for panda datasets from the lfc path
-    they will be registered in.
-    Method imported from DQ2.
-    """
 
-    suffixes_to_drop = ['_dis', '_sub', '_frag']
-    fields = dsn.split('.')
-    last_field = fields[-1]
-    try:
-        for suffix in suffixes_to_drop:
-            last_field = re.sub('%s.*$' % suffix, '', last_field)
-    except IndexError:
-        return dsn
-    fields[-1] = last_field
-    stripped_dsn = '.'.join(fields)
-    return stripped_dsn
+ScopeExtractionAlgorithmsT = TypeVar('ScopeExtractionAlgorithmsT', bound='ScopeExtractionAlgorithms')
 
 
-def __strip_tag(tag):
+class ScopeExtractionAlgorithms(PolicyPackageAlgorithms):
     """
-    Drop the _sub and _dis suffixes for panda datasets from the lfc path
-    they will be registered in
-    Method imported from DQ2.
+    Handle scope extraction algorithms
     """
-    suffixes_to_drop = ['_dis', '_sub', '_tid']
-    stripped_tag = tag
-    try:
-        for suffix in suffixes_to_drop:
-            stripped_tag = re.sub('%s.*$' % suffix, '', stripped_tag)
-    except IndexError:
-        return stripped_tag
-    return stripped_tag
 
+    _algorithm_type = 'scope'
 
-def clean_surls(surls):
-    res = []
-    for surl in surls:
-        if surl.startswith('srm'):
-            surl = re.sub(':[0-9]+/', '/', surl)
-            surl = re.sub(r'/srm/managerv1\?SFN=', '', surl)
-            surl = re.sub(r'/srm/v2/server\?SFN=', '', surl)
-            surl = re.sub(r'/srm/managerv2\?SFN=', '', surl)
-        if '?GoogleAccessId' in surl:
-            surl = surl.split('?GoogleAccessId')[0]
-        if '?X-Amz' in surl:
-            surl = surl.split('?X-Amz')[0]
-        res.append(surl)
-    res.sort()
-    return res
+    def __init__(self) -> None:
+        """
+        Initialises scope extraction algorithms object
+        """
+        super().__init__()
 
+    def extract_scope(self, did: str, scopes: Optional[Sequence[str]], extract_scope_convention: str) -> Sequence[str]:
+        """
+        Calls the correct algorithm for scope extraction
+        """
+        return self.get_algorithm(extract_scope_convention)(did, scopes)
 
-_EXTRACT_SCOPE_ALGORITHMS = {}
-_DEFAULT_EXTRACT = 'atlas'
-_loaded_policy_package_scope_algorithms = False
-
-
-def extract_scope_atlas(did, scopes):
-    # Try to extract the scope from the DSN
-    if did.find(':') > -1:
-        if len(did.split(':')) > 2:
-            raise RucioException('Too many colons. Cannot extract scope and name')
-        scope, name = did.split(':')[0], did.split(':')[1]
-        if name.endswith('/'):
-            name = name[:-1]
-        return scope, name
-    else:
-        scope = did.split('.')[0]
-        if did.startswith('user') or did.startswith('group'):
-            scope = ".".join(did.split('.')[0:2])
-        if did.endswith('/'):
-            did = did[:-1]
-        return scope, did
+    @classmethod
+    def supports(cls: type[ScopeExtractionAlgorithmsT], extract_scope_convention: str) -> bool:
+        """
+        Checks whether the specified scope extraction algorithm is supported
+        """
+        return super()._supports(cls._algorithm_type, extract_scope_convention)
 
+    @classmethod
+    def _module_init_(cls: type[ScopeExtractionAlgorithmsT]) -> None:
+        """
+        Registers the included scope extraction algorithms
+        """
+        cls.register('atlas', cls.extract_scope_atlas)
+        cls.register('belleii', cls.extract_scope_belleii)
+        cls.register('dirac', cls.extract_scope_dirac)
 
-def extract_scope_dirac(did, scopes):
-    # Default dirac scope extract algorithm. Scope is the second element in the LFN or the first one (VO name)
-    # if only one element is the result of a split.
-    elem = did.rstrip('/').split('/')
-    if len(elem) > 2:
-        scope = elem[2]
-    else:
-        scope = elem[1]
-    return scope, did
-
-
-def extract_scope_belleii(did, scopes):
-    split_did = did.split('/')
-    if did.startswith('/belle/mock/'):
-        return 'mock', did
-    if did.startswith('/belle/MC/'):
-        if did.startswith('/belle/MC/BG') or \
-           did.startswith('/belle/MC/build') or \
-           did.startswith('/belle/MC/generic') or \
-           did.startswith('/belle/MC/log') or \
-           did.startswith('/belle/MC/mcprod') or \
-           did.startswith('/belle/MC/prerelease') or \
-           did.startswith('/belle/MC/release'):
-            return 'mc', did
-        if did.startswith('/belle/MC/cert') or \
-           did.startswith('/belle/MC/dirac') or \
-           did.startswith('/belle/MC/dr3') or \
-           did.startswith('/belle/MC/fab') or \
-           did.startswith('/belle/MC/hideki') or \
-           did.startswith('/belle/MC/merge') or \
-           did.startswith('/belle/MC/migration') or \
-           did.startswith('/belle/MC/skim') or \
-           did.startswith('/belle/MC/test'):
-            return 'mc_tmp', did
-        if len(split_did) > 4:
-            if split_did[3].find('fab') > -1 or split_did[3].find('merge') > -1 or split_did[3].find('skim') > -1:
-                return 'mc_tmp', did
-            if split_did[3].find('release') > -1:
-                return 'mc', did
-        return 'mc_tmp', did
-    if did.startswith('/belle/Raw/'):
-        return 'raw', did
-    if did.startswith('/belle/hRaw'):
-        return 'hraw', did
-    if did.startswith('/belle/user/'):
-        if len(split_did) > 4:
-            if len(split_did[3]) == 1 and 'user.%s' % (split_did[4]) in scopes:
-                return 'user.%s' % split_did[4], did
-        if len(split_did) > 3:
-            if 'user.%s' % (split_did[3]) in scopes:
-                return 'user.%s' % split_did[3], did
-        return 'user', did
-    if did.startswith('/belle/group/'):
-        if len(split_did) > 4:
-            if 'group.%s' % (split_did[4]) in scopes:
-                return 'group.%s' % split_did[4], did
-        return 'group', did
-    if did.startswith('/belle/data/') or did.startswith('/belle/Data/'):
-        if len(split_did) > 4:
-            if split_did[3] in ['fab', 'skim']:  # /belle/Data/fab --> data_tmp
-                return 'data_tmp', did
-            if split_did[3].find('release') > -1:  # /belle/Data/release --> data
-                return 'data', did
-        if len(split_did) > 5:
-            if split_did[3] in ['proc']:  # /belle/Data/proc
-                if split_did[4].find('release') > -1:  # /belle/Data/proc/release*
-                    if len(split_did) > 7 and split_did[6] in ['GCR2c', 'prod00000007', 'prod6b', 'proc7b',
-                                                               'proc8b', 'Bucket4', 'Bucket6test', 'bucket6',
-                                                               'proc9', 'bucket7', 'SKIMDATAx1', 'proc10Valid',
-                                                               'proc10', 'SkimP10x1', 'SkimP11x1', 'SkimB9x1',
-                                                               'SkimB10x1', 'SkimB11x1']:  # /belle/Data/proc/release*/*/proc10/* --> data_tmp (Old convention)
-                        return 'data_tmp', did
-                    else:  # /belle/Data/proc/release*/*/proc11/* --> data (New convention)
-                        return 'data', did
-                if split_did[4].find('fab') > -1:  # /belle/Data/proc/fab* --> data_tmp
-                    return 'data_tmp', did
-        return 'data_tmp', did
-    if did.startswith('/belle/ddm/functional_tests/') or did.startswith('/belle/ddm/tests/') or did.startswith('/belle/test/ddm_test'):
-        return 'test', did
-    if did.startswith('/belle/BG/'):
-        return 'data', did
-    if did.startswith('/belle/collection'):
-        return 'collection', did
-    return 'other', did
+    @classmethod
+    def get_algorithm(cls: type[ScopeExtractionAlgorithmsT], extract_scope_convention: str) -> Callable[[str, Optional[Sequence[str]]], Sequence[str]]:
+        """
+        Looks up a scope extraction algorithm by name
+        """
+        return super()._get_one_algorithm(cls._algorithm_type, extract_scope_convention)
 
+    @classmethod
+    def register(cls: type[ScopeExtractionAlgorithmsT], name: str, fn_extract_scope: Callable[[str, Optional[Sequence[str]]], Sequence[str]]) -> None:
+        """
+        Registers a new scope extraction algorithm
+        """
+        algorithm_dict = {name: fn_extract_scope}
+        super()._register(cls._algorithm_type, algorithm_dict)
+
+    @staticmethod
+    def extract_scope_atlas(did: str, scopes: Optional[Sequence[str]]) -> Sequence[str]:
+        # Try to extract the scope from the DSN
+        if did.find(':') > -1:
+            if len(did.split(':')) > 2:
+                raise RucioException('Too many colons. Cannot extract scope and name')
+            scope, name = did.split(':')[0], did.split(':')[1]
+            if name.endswith('/'):
+                name = name[:-1]
+            return scope, name
+        else:
+            scope = did.split('.')[0]
+            if did.startswith('user') or did.startswith('group'):
+                scope = ".".join(did.split('.')[0:2])
+            if did.endswith('/'):
+                did = did[:-1]
+            return scope, did
+
+    @staticmethod
+    def extract_scope_dirac(did: str, scopes: Optional[Sequence[str]]) -> Sequence[str]:
+        # Default dirac scope extract algorithm. Scope is the second element in the LFN or the first one (VO name)
+        # if only one element is the result of a split.
+        elem = did.rstrip('/').split('/')
+        if len(elem) > 2:
+            scope = elem[2]
+        else:
+            scope = elem[1]
+        return scope, did
 
-def register_extract_scope_algorithm(extract_callable, name=[]):
-    if name is None:
-        name = extract_callable.__name__
-    _EXTRACT_SCOPE_ALGORITHMS[name] = extract_callable
+    @staticmethod
+    def extract_scope_belleii(did: str, scopes: Optional[Sequence[str]]) -> Sequence[str]:
+        split_did = did.split('/')
+        if did.startswith('/belle/mock/'):
+            return 'mock', did
+        if did.startswith('/belle/MC/'):
+            if did.startswith('/belle/MC/BG') or \
+               did.startswith('/belle/MC/build') or \
+               did.startswith('/belle/MC/generic') or \
+               did.startswith('/belle/MC/log') or \
+               did.startswith('/belle/MC/mcprod') or \
+               did.startswith('/belle/MC/prerelease') or \
+               did.startswith('/belle/MC/release'):
+                return 'mc', did
+            if did.startswith('/belle/MC/cert') or \
+               did.startswith('/belle/MC/dirac') or \
+               did.startswith('/belle/MC/dr3') or \
+               did.startswith('/belle/MC/fab') or \
+               did.startswith('/belle/MC/hideki') or \
+               did.startswith('/belle/MC/merge') or \
+               did.startswith('/belle/MC/migration') or \
+               did.startswith('/belle/MC/skim') or \
+               did.startswith('/belle/MC/test'):
+                return 'mc_tmp', did
+            if len(split_did) > 4:
+                if split_did[3].find('fab') > -1 or split_did[3].find('merge') > -1 or split_did[3].find('skim') > -1:
+                    return 'mc_tmp', did
+                if split_did[3].find('release') > -1:
+                    return 'mc', did
+            return 'mc_tmp', did
+        if did.startswith('/belle/Raw/'):
+            return 'raw', did
+        if did.startswith('/belle/hRaw'):
+            return 'hraw', did
+        if did.startswith('/belle/user/'):
+            if len(split_did) > 4:
+                if len(split_did[3]) == 1 and scopes is not None and 'user.%s' % (split_did[4]) in scopes:
+                    return 'user.%s' % split_did[4], did
+            if len(split_did) > 3:
+                if scopes is not None and 'user.%s' % (split_did[3]) in scopes:
+                    return 'user.%s' % split_did[3], did
+            return 'user', did
+        if did.startswith('/belle/group/'):
+            if len(split_did) > 4:
+                if scopes is not None and 'group.%s' % (split_did[4]) in scopes:
+                    return 'group.%s' % split_did[4], did
+            return 'group', did
+        if did.startswith('/belle/data/') or did.startswith('/belle/Data/'):
+            if len(split_did) > 4:
+                if split_did[3] in ['fab', 'skim']:  # /belle/Data/fab --> data_tmp
+                    return 'data_tmp', did
+                if split_did[3].find('release') > -1:  # /belle/Data/release --> data
+                    return 'data', did
+            if len(split_did) > 5:
+                if split_did[3] in ['proc']:  # /belle/Data/proc
+                    if split_did[4].find('release') > -1:  # /belle/Data/proc/release*
+                        if len(split_did) > 7 and split_did[6] in ['GCR2c', 'prod00000007', 'prod6b', 'proc7b',
+                                                                   'proc8b', 'Bucket4', 'Bucket6test', 'bucket6',
+                                                                   'proc9', 'bucket7', 'SKIMDATAx1', 'proc10Valid',
+                                                                   'proc10', 'SkimP10x1', 'SkimP11x1', 'SkimB9x1',
+                                                                   'SkimB10x1', 'SkimB11x1']:  # /belle/Data/proc/release*/*/proc10/* --> data_tmp (Old convention)
+                            return 'data_tmp', did
+                        else:  # /belle/Data/proc/release*/*/proc11/* --> data (New convention)
+                            return 'data', did
+                    if split_did[4].find('fab') > -1:  # /belle/Data/proc/fab* --> data_tmp
+                        return 'data_tmp', did
+            return 'data_tmp', did
+        if did.startswith('/belle/ddm/functional_tests/') or did.startswith('/belle/ddm/tests/') or did.startswith('/belle/test/ddm_test'):
+            return 'test', did
+        if did.startswith('/belle/BG/'):
+            return 'data', did
+        if did.startswith('/belle/collection'):
+            return 'collection', did
+        return 'other', did
 
 
-register_extract_scope_algorithm(extract_scope_atlas, 'atlas')
-register_extract_scope_algorithm(extract_scope_belleii, 'belleii')
-register_extract_scope_algorithm(extract_scope_dirac, 'dirac')
+_DEFAULT_EXTRACT = 'atlas'
+ScopeExtractionAlgorithms._module_init_()
 
 
-def extract_scope(did, scopes=None, default_extract=_DEFAULT_EXTRACT):
-    global _loaded_policy_package_scope_algorithms
-    if not _loaded_policy_package_scope_algorithms:
-        register_policy_package_algorithms('scope', _EXTRACT_SCOPE_ALGORITHMS)
-        _loaded_policy_package_scope_algorithms = True
+def extract_scope(
+        did: str,
+        scopes: Optional[Sequence[str]] = None,
+        default_extract: str = _DEFAULT_EXTRACT
+) -> Sequence[str]:
+    scope_extraction_algorithms = ScopeExtractionAlgorithms()
     extract_scope_convention = config_get('common', 'extract_scope', False, None) or config_get('policy', 'extract_scope', False, None)
-    if extract_scope_convention is None or extract_scope_convention not in _EXTRACT_SCOPE_ALGORITHMS:
+    if extract_scope_convention is None or not ScopeExtractionAlgorithms.supports(extract_scope_convention):
         extract_scope_convention = default_extract
-    return _EXTRACT_SCOPE_ALGORITHMS[extract_scope_convention](did=did, scopes=scopes)
+    return scope_extraction_algorithms.extract_scope(did, scopes, extract_scope_convention)
 
 
-def pid_exists(pid):
+def pid_exists(pid: int) -> bool:
     """
     Check whether pid exists in the current process table.
     UNIX only.
@@ -835,7 +1134,7 @@ def pid_exists(pid):
         return True
 
 
-def sizefmt(num, human=True):
+def sizefmt(num: Union[int, float, None], human: bool = True) -> str:
     """
     Print human readable file sizes
     """
@@ -855,7 +1154,7 @@ def sizefmt(num, human=True):
         return 'Inf'
 
 
-def get_tmp_dir():
+def get_tmp_dir() -> str:
     """
     Get a path where to store temporary files.
 
@@ -883,7 +1182,7 @@ def get_tmp_dir():
     return base_dir
 
 
-def is_archive(name):
+def is_archive(name: str) -> bool:
     '''
     Check if a file name is an archive file or not.
 
@@ -908,7 +1207,28 @@ class Color:
     END = '\033[0m'
 
 
-def detect_client_location():
+def resolve_ips(hostname: str) -> list[str]:
+    try:
+        ipaddress.ip_address(hostname)
+        return [hostname]
+    except ValueError:
+        pass
+    try:
+        addrinfo = socket.getaddrinfo(hostname, 0, socket.AF_INET, 0, socket.IPPROTO_TCP)
+        return [ai[4][0] for ai in addrinfo]
+    except socket.gaierror:
+        pass
+    return []
+
+
+def resolve_ip(hostname: str) -> str:
+    ips = resolve_ips(hostname)
+    if ips:
+        return ips[0]
+    return hostname
+
+
+def detect_client_location() -> "IPDict":
     """
     Normally client IP will be set on the server side (request.remote_addr)
     Here setting ip on the one seen by the host itself. There is no connection
@@ -923,22 +1243,22 @@ def detect_client_location():
     ip = None
 
     try:
-        s = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)
-        s.connect(("2001:4860:4860:0:0:0:0:8888", 80))
-        ip = s.getsockname()[0]
+        with socket.socket(socket.AF_INET6, socket.SOCK_DGRAM) as s:
+            s.connect(("2001:4860:4860:0:0:0:0:8888", 80))
+            ip = s.getsockname()[0]
     except Exception:
         pass
 
     if not ip:
         try:
-            s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
-            s.connect(("8.8.8.8", 80))
-            ip = s.getsockname()[0]
+            with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as s:
+                s.connect(("8.8.8.8", 80))
+                ip = s.getsockname()[0]
         except Exception:
             pass
 
     if not ip:
-        ip = '0.0.0.0'
+        ip = '0.0.0.0'  # noqa: S104
 
     site = os.environ.get('SITE_NAME',
                           os.environ.get('ATLAS_SITE_NAME',
@@ -964,7 +1284,7 @@ def detect_client_location():
             'longitude': longitude}
 
 
-def ssh_sign(private_key, message):
+def ssh_sign(private_key: str, message: str) -> str:
     """
     Sign a string message using the private key.
 
@@ -972,21 +1292,20 @@ def ssh_sign(private_key, message):
     :param message: The message to sign as a string.
     :return: Base64 encoded signature as a string.
     """
-    if isinstance(message, str):
-        message = message.encode()
+    encoded_message = message.encode()
     if not EXTRA_MODULES['paramiko']:
         raise MissingModuleException('The paramiko module is not installed or faulty.')
     sio_private_key = StringIO(private_key)
     priv_k = RSAKey.from_private_key(sio_private_key)
     sio_private_key.close()
-    signature_stream = priv_k.sign_ssh_data(message)
+    signature_stream = priv_k.sign_ssh_data(encoded_message)
     signature_stream.rewind()
     base64_encoded = base64.b64encode(signature_stream.get_remainder())
     base64_encoded = base64_encoded.decode()
     return base64_encoded
 
 
-def make_valid_did(lfn_dict):
+def make_valid_did(lfn_dict: dict[str, Any]) -> dict[str, Any]:
     """
     When managing information about a LFN (such as in `rucio upload` or
     the RSE manager's upload), we add the `filename` attribute to record
@@ -1006,7 +1325,7 @@ def make_valid_did(lfn_dict):
     return lfn_copy
 
 
-def send_trace(trace, trace_endpoint, user_agent, retries=5):
+def send_trace(trace: TraceDict, trace_endpoint: str, user_agent: str, retries: int = 5) -> int:
     """
     Send the given trace to the trace endpoint
 
@@ -1027,7 +1346,7 @@ def send_trace(trace, trace_endpoint, user_agent, retries=5):
     return 1
 
 
-def add_url_query(url, query):
+def add_url_query(url: str, query: dict[str, str]) -> str:
     """
     Add a new dictionary to URL parameters
 
@@ -1043,7 +1362,7 @@ def add_url_query(url, query):
     return urlunparse(url_parts)
 
 
-def get_bytes_value_from_string(input_string):
+def get_bytes_value_from_string(input_string: str) -> Union[bool, int]:
     """
     Get bytes from a string that represents a storage value and unit
 
@@ -1073,7 +1392,7 @@ def get_bytes_value_from_string(input_string):
         return False
 
 
-def parse_did_filter_from_string(input_string):
+def parse_did_filter_from_string(input_string: str) -> tuple[dict[str, Any], str]:
     """
     Parse DID filter options in format 'length<3,type=all' from string.
 
@@ -1110,13 +1429,13 @@ def parse_did_filter_from_string(input_string):
                     value = datetime.datetime.strptime(value, '%Y-%m-%dT%H:%M:%S.%fZ')
 
             if key == 'type':
-                if value.upper() in ['ALL', 'COLLECTION', 'CONTAINER', 'DATASET', 'FILE']:
-                    type_ = value.lower()
+                if value.upper() in ['ALL', 'COLLECTION', 'CONTAINER', 'DATASET', 'FILE']:  # type: ignore
+                    type_ = value.lower()  # type: ignore
                 else:
                     raise InvalidType('{0} is not a valid type. Valid types are {1}'.format(value, ['ALL', 'COLLECTION', 'CONTAINER', 'DATASET', 'FILE']))
             elif key in ('length.gt', 'length.lt', 'length.gte', 'length.lte', 'length'):
                 try:
-                    value = int(value)
+                    value = int(value)  # type: ignore
                     filters[key] = value
                 except ValueError:
                     raise ValueError('Length has to be an integer value.')
@@ -1133,7 +1452,12 @@ def parse_did_filter_from_string(input_string):
     return filters, type_
 
 
-def parse_did_filter_from_string_fe(input_string, name='*', type='collection', omit_name=False):
+def parse_did_filter_from_string_fe(
+        input_string: str,
+        name: str = '*',
+        type: str = 'collection',
+        omit_name: bool = False
+) -> tuple[list[dict[str, Any]], str]:
     """
     Parse DID filter string for the filter engine (fe).
 
@@ -1241,7 +1565,7 @@ def parse_did_filter_from_string_fe(input_string, name='*', type='collection', o
     return filters, type
 
 
-def parse_replicas_from_file(path):
+def parse_replicas_from_file(path: "FileDescriptorOrPath") -> Any:
     """
     Parses the output of list_replicas from a json or metalink file
     into a dictionary. Metalink parsing is tried first and if it fails
@@ -1253,7 +1577,7 @@ def parse_replicas_from_file(path):
     """
     with open(path) as fp:
         try:
-            root = ElementTree.parse(fp).getroot()
+            root = ElementTree.parse(fp).getroot()  # noqa: S314
             return parse_replicas_metalink(root)
         except ElementTree.ParseError as xml_err:
             try:
@@ -1262,7 +1586,7 @@ def parse_replicas_from_file(path):
                 raise MetalinkJsonParsingError(path, xml_err, json_err)
 
 
-def parse_replicas_from_string(string):
+def parse_replicas_from_string(string: str) -> Any:
     """
     Parses the output of list_replicas from a json or metalink string
     into a dictionary. Metalink parsing is tried first and if it fails
@@ -1273,7 +1597,7 @@ def parse_replicas_from_string(string):
     :returns: a list with a dictionary for each file
     """
     try:
-        root = ElementTree.fromstring(string)
+        root = ElementTree.fromstring(string)  # noqa: S314
         return parse_replicas_metalink(root)
     except ElementTree.ParseError as xml_err:
         try:
@@ -1282,7 +1606,7 @@ def parse_replicas_from_string(string):
             raise MetalinkJsonParsingError(string, xml_err, json_err)
 
 
-def parse_replicas_metalink(root):
+def parse_replicas_metalink(root: ElementTree.Element) -> list[dict[str, Any]]:
     """
     Transforms the metalink tree into a list of dictionaries where
     each dictionary describes a file with its replicas.
@@ -1339,11 +1663,15 @@ def parse_replicas_metalink(root):
     return files
 
 
-def get_thread_with_periodic_running_function(interval, action, graceful_stop):
+def get_thread_with_periodic_running_function(
+        interval: Union[int, float],
+        action: Callable[..., Any],
+        graceful_stop: threading.Event
+) -> threading.Thread:
     """
     Get a thread where a function runs periodically.
 
-    :param interval: Interval in seconds when the action fucntion should run.
+    :param interval: Interval in seconds when the action function should run.
     :param action: Function, that should run periodically.
     :param graceful_stop: Threading event used to check for graceful stop.
     """
@@ -1351,12 +1679,12 @@ def get_thread_with_periodic_running_function(interval, action, graceful_stop):
         while not graceful_stop.is_set():
             starttime = time.time()
             action()
-            time.sleep(interval - ((time.time() - starttime)))
+            time.sleep(interval - (time.time() - starttime))
     t = threading.Thread(target=start)
     return t
 
 
-def run_cmd_process(cmd, timeout=3600):
+def run_cmd_process(cmd: str, timeout: int = 3600) -> tuple[int, str]:
     """
     shell command parser with timeout
 
@@ -1397,7 +1725,10 @@ def run_cmd_process(cmd, timeout=3600):
     return returncode, stdout
 
 
-def api_update_return_dict(dictionary, session=None):
+def gateway_update_return_dict(
+        dictionary: dict[str, Any],
+        session: Optional["Session"] = None
+) -> dict[str, Any]:
     """
     Ensure that rse is in a dictionary returned from core
 
@@ -1435,7 +1766,12 @@ def api_update_return_dict(dictionary, session=None):
     return dictionary
 
 
-def setup_logger(module_name=None, logger_name=None, logger_level=None, verbose=False):
+def setup_logger(
+        module_name: Optional[str] = None,
+        logger_name: Optional[str] = None,
+        logger_level: Optional[int] = None,
+        verbose: bool = False
+) -> logging.Logger:
     '''
     Factory method to set logger with handlers.
     :param module_name: __name__ of the module that is calling this method
@@ -1444,10 +1780,10 @@ def setup_logger(module_name=None, logger_name=None, logger_level=None, verbose=
     :param verbose: verbose option set in bin/rucio
     '''
     # helper method for cfg check
-    def _force_cfg_log_level(cfg_option):
+    def _force_cfg_log_level(cfg_option: str) -> bool:
         cfg_forced_modules = config_get('logging', cfg_option, raise_exception=False, default=None, clean_cached=True,
                                         check_config_table=False)
-        if cfg_forced_modules:
+        if cfg_forced_modules and module_name is not None:
             if re.match(str(cfg_forced_modules), module_name):
                 return True
         return False
@@ -1477,11 +1813,11 @@ def setup_logger(module_name=None, logger_name=None, logger_level=None, verbose=
     logger.setLevel(logger_level)
 
     # preferred logger handling
-    def add_handler(logger):
+    def add_handler(logger: logging.Logger) -> None:
         hdlr = logging.StreamHandler()
 
-        def emit_decorator(fnc):
-            def func(*args):
+        def emit_decorator(fnc: Callable[..., Any]) -> Callable[..., Any]:
+            def func(*args) -> Callable[..., Any]:
                 if 'RUCIO_LOGGING_FORMAT' not in os.environ:
                     levelno = args[0].levelno
                     format_str = '%(asctime)s\t%(levelname)s\t%(message)s\033[0m'
@@ -1514,7 +1850,12 @@ def setup_logger(module_name=None, logger_name=None, logger_level=None, verbose=
     return logger
 
 
-def daemon_sleep(start_time, sleep_time, graceful_stop, logger=logging.log):
+def daemon_sleep(
+        start_time: float,
+        sleep_time: float,
+        graceful_stop: threading.Event,
+        logger: "LoggerFunction" = logging.log
+) -> None:
     """Sleeps a daemon the time provided by sleep_time"""
     end_time = time.time()
     time_diff = end_time - start_time
@@ -1523,7 +1864,7 @@ def daemon_sleep(start_time, sleep_time, graceful_stop, logger=logging.log):
         graceful_stop.wait(sleep_time - time_diff)
 
 
-def is_client():
+def is_client() -> bool:
     """"
     Checks if the function is called from a client or from a server/daemon
 
@@ -1537,7 +1878,7 @@ def is_client():
                 client_mode = True
             else:
                 client_mode = False
-        except RuntimeError:
+        except (RuntimeError, ConfigNotFound):
             # If no configuration file is found the default value should be True
             client_mode = True
     else:
@@ -1552,15 +1893,15 @@ def is_client():
 class retry:
     """Retry callable object with configuragle number of attempts"""
 
-    def __init__(self, func, *args, **kwargs):
+    def __init__(self, func: Callable[..., Any], *args, **kwargs):
         '''
         :param func: a method that should be executed with retries
-        :param args: parametres of the func
+        :param args: parameters of the func
         :param kwargs: key word arguments of the func
         '''
         self.func, self.args, self.kwargs = func, args, kwargs
 
-    def __call__(self, mtries=3, logger=logging.log):
+    def __call__(self, mtries: int = 3, logger: "LoggerFunction" = logging.log) -> Callable[..., Any]:
         '''
         :param mtries: maximum number of attempts to execute the function
         :param logger: preferred logger
@@ -1586,9 +1927,9 @@ class StoreAndDeprecateWarningAction(argparse.Action):
     '''
 
     def __init__(self,
-                 option_strings,
-                 new_option_string,
-                 dest,
+                 option_strings: Sequence[str],
+                 new_option_string: str,
+                 dest: str,
                  **kwargs):
         """
         :param option_strings: all possible argument name strings
@@ -1600,10 +1941,11 @@ class StoreAndDeprecateWarningAction(argparse.Action):
             option_strings=option_strings,
             dest=dest,
             **kwargs)
-        assert new_option_string in option_strings
+        if new_option_string not in option_strings:
+            raise ValueError("%s not supported as a string option." % new_option_string)
         self.new_option_string = new_option_string
 
-    def __call__(self, parser, namespace, values, option_string=None):
+    def __call__(self, parser, namespace, values, option_string: Optional[str] = None):
         if option_string and option_string != self.new_option_string:
             # The logger gets typically initialized after the argument parser
             # to set the verbosity of the logger. Thus using simple print to console.
@@ -1619,12 +1961,12 @@ class StoreTrueAndDeprecateWarningAction(argparse._StoreConstAction):
     '''
 
     def __init__(self,
-                 option_strings,
-                 new_option_string,
-                 dest,
-                 default=False,
-                 required=False,
-                 help=None):
+                 option_strings: Sequence[str],
+                 new_option_string: str,
+                 dest: str,
+                 default: bool = False,
+                 required: bool = False,
+                 help: Optional[str] = None):
         """
         :param option_strings: all possible argument name strings
         :param new_option_string: the new option string which replaces the old
@@ -1638,10 +1980,11 @@ class StoreTrueAndDeprecateWarningAction(argparse._StoreConstAction):
             default=default,
             required=required,
             help=help)
-        assert new_option_string in option_strings
+        if new_option_string not in option_strings:
+            raise ValueError("%s not supported as a string option." % new_option_string)
         self.new_option_string = new_option_string
 
-    def __call__(self, parser, namespace, values, option_string=None):
+    def __call__(self, parser, namespace, values, option_string: Optional[str] = None):
         super(StoreTrueAndDeprecateWarningAction, self).__call__(parser, namespace, values, option_string=option_string)
         if option_string and option_string != self.new_option_string:
             # The logger gets typically initialized after the argument parser
@@ -1660,7 +2003,7 @@ class PriorityQueue:
     [1] https://en.wikipedia.org/wiki/Heap_(data_structure)
     """
     class ContainerSlot:
-        def __init__(self, position, priority):
+        def __init__(self, position: int, priority: int):
             self.pos = position
             self.prio = priority
 
@@ -1752,78 +2095,9 @@ class PriorityQueue:
         return heap_changed
 
 
-def register_policy_package_algorithms(algorithm_type, dictionary):
-    '''
-    Loads all the algorithms of a given type from the policy package(s) and registers them
-    :param algorithm_type: the type of algorithm to register (e.g. 'surl', 'lfn2pfn')
-    :param dictionary: the dictionary to register them in
-    :param vo: the name of the relevant VO (None for single VO)
-    '''
-    def try_importing_policy(algorithm_type, dictionary, vo=None):
-        import importlib
-        try:
-            env_name = 'RUCIO_POLICY_PACKAGE' + ('' if not vo else '_' + vo.upper())
-            if env_name in os.environ:
-                package = os.environ[env_name]
-            else:
-                package = config.config_get('policy', 'package' + ('' if not vo else '-' + vo))
-            check_policy_package_version(package)
-            module = importlib.import_module(package)
-            if hasattr(module, 'get_algorithms'):
-                all_algorithms = module.get_algorithms()
-                if algorithm_type in all_algorithms:
-                    algorithms = all_algorithms[algorithm_type]
-                    if not vo:
-                        dictionary.update(algorithms)
-                    else:
-                        # check that the names are correctly prefixed
-                        for k in algorithms.keys():
-                            if k.lower().startswith(vo.lower()):
-                                dictionary[k] = algorithms[k]
-                            else:
-                                raise InvalidAlgorithmName(k, vo)
-        except (NoOptionError, NoSectionError, ImportError):
-            pass
-
-    from rucio.common import config
-    try:
-        multivo = config.config_get_bool('common', 'multi_vo')
-    except (NoOptionError, NoSectionError):
-        multivo = False
-    if not multivo:
-        # single policy package
-        try_importing_policy(algorithm_type, dictionary)
-    else:
-        # determine whether on client or server
-        client = False
-        if 'RUCIO_CLIENT_MODE' not in os.environ:
-            if not config.config_has_section('database') and config.config_has_section('client'):
-                client = True
-        else:
-            if os.environ['RUCIO_CLIENT_MODE']:
-                client = True
-
-        # on client, only register algorithms for selected VO
-        if client:
-            if 'RUCIO_VO' in os.environ:
-                vo = os.environ['RUCIO_VO']
-            else:
-                try:
-                    vo = config.config_get('client', 'vo')
-                except (NoOptionError, NoSectionError):
-                    vo = 'def'
-            try_importing_policy(algorithm_type, dictionary, vo)
-        # on server, list all VOs and register their algorithms
-        else:
-            from rucio.core.vo import list_vos
-            # policy package per VO
-            vos = list_vos()
-            for vo in vos:
-                try_importing_policy(algorithm_type, dictionary, vo['vo'])
-
-
-def check_policy_package_version(package):
+def check_policy_package_version(package: str) -> None:
     import importlib
+
     from rucio.version import version_string
     '''
     Checks that the Rucio version supported by the policy package is compatible
@@ -1842,13 +2116,13 @@ def check_policy_package_version(package):
     components = 2 if version_string().startswith("1.") else 1
     current_version = ".".join(version_string().split(".")[:components])
     if current_version not in supported_version:
-        raise PolicyPackageVersionError(package)
+        raise PolicyPackageVersionError(package, current_version, supported_version)
 
 
 class Availability:
     """
     This util class acts as a translator between the availability stored as
-    integer and as boolen values.
+    integer and as boolean values.
 
     `None` represents a missing value. This lets a user update a specific value
     without altering the other ones. If it needs to be evaluated, it will
@@ -1859,7 +2133,12 @@ class Availability:
     write = None
     delete = None
 
-    def __init__(self, read=None, write=None, delete=None):
+    def __init__(
+            self,
+            read: Optional[bool] = None,
+            write: Optional[bool] = None,
+            delete: Optional[bool] = None
+    ):
         self.read = read
         self.write = write
         self.delete = delete
@@ -1944,3 +2223,16 @@ def retrying(
                 time.sleep(wait_fixed / 1000.0)
         return _wrapper
     return _decorator
+
+
+def deep_merge_dict(source: dict, destination: dict) -> dict:
+    """Merge two dictionaries together recursively"""
+    for key, value in source.items():
+        if isinstance(value, dict):
+            # get node or create one
+            node = destination.setdefault(key, {})
+            deep_merge_dict(value, node)
+        else:
+            destination[key] = value
+
+    return destination