rucio 33.6.1__tar.gz → 34.0.0rc2__tar.gz

{rucio-33.6.1 → rucio-34.0.0rc2}/AUTHORS.rst

@@ -65,6 +65,7 @@ Individual contributors to the source code
 - Stefan Piperov <stefan.piperov@cern.ch>, 2021
 - Jensen Zhang <hack@jensen-zhang.site>, 2022
 - Aksel Lunde Aase <aksel.lunde.aase@gmail.com>, 2022
+- Elena Gazzarrini <gazzarrini.elena@gmail.com>, 2022-2023
 
 Organisations employing contributors
 ------------------------------------

{rucio-33.6.1 → rucio-34.0.0rc2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: rucio
-Version: 33.6.1
+Version: 34.0.0rc2
 Summary: Rucio Package
 Home-page: https://rucio.cern.ch/
 Author: Rucio

{rucio-33.6.1 → rucio-34.0.0rc2}/lib/rucio/api/did.py

@@ -22,8 +22,9 @@ from rucio.common.exception import RucioException
 from rucio.common.schema import validate_schema
 from rucio.common.types import InternalAccount, InternalScope
 from rucio.common.utils import api_update_return_dict
-from rucio.core import did, naming_convention, meta as meta_core
+from rucio.core import did, naming_convention
 from rucio.core.rse import get_rse_id
+from rucio.core import meta_conventions as meta_convention_core
 from rucio.db.sqla.constants import DIDType
 from rucio.db.sqla.session import read_session, stream_session, transactional_session
 
@@ -117,7 +118,7 @@ def add_did(scope, name, did_type, issuer, account=None, statuses={}, meta={}, r
                 raise rucio.common.exception.InvalidObject("Provided metadata %s doesn't match the naming convention: %s != %s" % (k, meta[k], extra_meta[k]))
 
         # Validate metadata
-        meta_core.validate_meta(meta=meta, did_type=DIDType[did_type.upper()], session=session)
+        meta_convention_core.validate_meta(meta=meta, did_type=DIDType[did_type.upper()], session=session)
 
     return did.add_did(scope=scope, name=name, did_type=DIDType[did_type.upper()], account=account or issuer,
                        statuses=statuses, meta=meta, rules=rules, lifetime=lifetime,

rucio-33.6.1/lib/rucio/api/meta.py → rucio-34.0.0rc2/lib/rucio/api/meta_conventions.py

@@ -13,33 +13,35 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Union, Optional
 
 from rucio.api.permission import has_permission
 from rucio.common.exception import AccessDenied
-from rucio.core import meta
+from rucio.core import meta_conventions
 from rucio.db.sqla.session import read_session, transactional_session
+from rucio.db.sqla.constants import KeyType
 
 if TYPE_CHECKING:
     from sqlalchemy.orm import Session
+    from rucio.common.types import InternalAccount
 
 
 @read_session
 def list_keys(*, session: "Session"):
     """
-    Lists all keys.
+    Lists all keys for DID Metadata Conventions.
 
     :param session: The database session in use.
 
     :returns: A list containing all keys.
     """
-    return meta.list_keys(session=session)
+    return meta_conventions.list_keys(session=session)
 
 
 @read_session
-def list_values(key, *, session: "Session"):
+def list_values(key: str, *, session: "Session"):
     """
-    Lists all values for a key.
+    Lists all allowed values for a DID key (all values for a key in DID Metadata Conventions).
 
     :param key: the name for the key.
     :param session: The database session in use.
@@ -47,13 +49,13 @@ def list_values(key, *, session: "Session"):
 
     :returns: A list containing all values.
     """
-    return meta.list_values(key=key, session=session)
+    return meta_conventions.list_values(key=key, session=session)
 
 
 @transactional_session
-def add_key(key, key_type, issuer, value_type=None, value_regexp=None, vo='def', *, session: "Session"):
+def add_key(key: str, key_type: Union[KeyType, str], issuer: "InternalAccount", value_type: Optional[str] = None, value_regexp: Optional[str] = None, vo: str = 'def', *, session: "Session"):
     """
-    Add a new allowed key.
+    Add an allowed key for DID metadata (update the DID Metadata Conventions table with a new key).
 
     :param key: the name for the new key.
     :param key_type: the type of the key: all(container, dataset, file), collection(dataset or container), file, derived(compute from file for collection).
@@ -66,13 +68,13 @@ def add_key(key, key_type, issuer, value_type=None, value_regexp=None, vo='def',
     kwargs = {'key': key, 'key_type': key_type, 'value_type': value_type, 'value_regexp': value_regexp}
     if not has_permission(issuer=issuer, vo=vo, action='add_key', kwargs=kwargs, session=session):
         raise AccessDenied('Account %s can not add key' % (issuer))
-    return meta.add_key(key=key, key_type=key_type, value_type=value_type, value_regexp=value_regexp, session=session)
+    return meta_conventions.add_key(key=key, key_type=key_type, value_type=value_type, value_regexp=value_regexp, session=session)
 
 
 @transactional_session
-def add_value(key, value, issuer, vo='def', *, session: "Session"):
+def add_value(key: str, value: str, issuer: "InternalAccount", vo: str = 'def', *, session: "Session"):
     """
-    Add a new value to a key.
+    Add an allowed value for DID metadata (update a key in DID Metadata Conventions table).
 
     :param key: the name for the key.
     :param value: the value.
@@ -82,4 +84,4 @@ def add_value(key, value, issuer, vo='def', *, session: "Session"):
     kwargs = {'key': key, 'value': value}
     if not has_permission(issuer=issuer, vo=vo, action='add_value', kwargs=kwargs, session=session):
         raise AccessDenied('Account %s can not add value %s to key %s' % (issuer, value, key))
-    return meta.add_value(key=key, value=value, session=session)
+    return meta_conventions.add_value(key=key, value=value, session=session)

{rucio-33.6.1 → rucio-34.0.0rc2}/lib/rucio/client/client.py

@@ -27,7 +27,7 @@ from rucio.client.exportclient import ExportClient
 from rucio.client.importclient import ImportClient
 from rucio.client.lifetimeclient import LifetimeClient
 from rucio.client.lockclient import LockClient
-from rucio.client.metaclient import MetaClient
+from rucio.client.metaconventionsclient import MetaConventionClient
 from rucio.client.pingclient import PingClient
 from rucio.client.replicaclient import ReplicaClient
 from rucio.client.requestclient import RequestClient
@@ -40,7 +40,7 @@ from rucio.client.touchclient import TouchClient
 
 class Client(AccountClient,
              AccountLimitClient,
-             MetaClient,
+             MetaConventionClient,
              PingClient,
              ReplicaClient,
              RequestClient,

rucio-33.6.1/lib/rucio/client/metaclient.py → rucio-34.0.0rc2/lib/rucio/client/metaconventionsclient.py

@@ -21,17 +21,19 @@ from requests.status_codes import codes
 from rucio.client.baseclient import BaseClient
 from rucio.client.baseclient import choice
 from rucio.common.utils import build_url
+from rucio.db.sqla.constants import KeyType
+from typing import Union, Optional
 
 
-class MetaClient(BaseClient):
+class MetaConventionClient(BaseClient):
 
-    """Meta client class for working with data identifier attributes"""
+    """Metadata client class for working with data identifier attributes"""
 
-    META_BASEURL = 'meta'
+    META_BASEURL = 'meta_conventions'
 
-    def add_key(self, key, key_type, value_type=None, value_regexp=None):
+    def add_key(self, key: str, key_type: Union[KeyType, str], value_type: Optional[str] = None, value_regexp: Optional[str] = None) -> Optional[bool]:
         """
-        Sends the request to add a new key.
+        Sends the request to add an allowed key for DID metadata (update the DID Metadata Conventions table with a new key).
 
         :param key: the name for the new key.
         :param key_type: the type of the key: all(container, dataset, file), collection(dataset or container), file, derived(compute from file for collection).
@@ -56,9 +58,9 @@ class MetaClient(BaseClient):
             exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
             raise exc_cls(exc_msg)
 
-    def list_keys(self):
+    def list_keys(self) -> Optional[list[str]]:
         """
-        Sends the request to list all keys.
+        Sends the request to list all keys for DID Metadata Conventions.
 
         :return: a list containing the names of all keys.
         """
@@ -72,9 +74,10 @@ class MetaClient(BaseClient):
             exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
             raise exc_cls(exc_msg)
 
-    def list_values(self, key):
+    def list_values(self, key: str) -> Optional[list[str]]:
         """
-        Sends the request to list all values for a key.
+        Sends the request to lists all allowed values for a DID key (all values for a key in DID Metadata Conventions).
+.
 
         :return: a list containing the names of all values for a key.
         """
@@ -88,9 +91,9 @@ class MetaClient(BaseClient):
             exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
             raise exc_cls(exc_msg)
 
-    def add_value(self, key, value):
+    def add_value(self, key: str, value: str) -> Optional[bool]:
         """
-        Sends the request to add a value to a key.
+        Sends the request to add a value for a key in DID Metadata Convention.
 
         :param key: the name for key.
         :param value: the value.
@@ -111,7 +114,7 @@ class MetaClient(BaseClient):
 
     def del_value(self, key, value):
         """
-        Delete a value for a key.
+        Delete a key in the DID Metadata Conventions table.
 
         :param key: the name for key.
         :param value: the value.

{rucio-33.6.1 → rucio-34.0.0rc2}/lib/rucio/client/uploadclient.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import base64
 import copy
 import json
 import logging
@@ -24,13 +25,13 @@ import time
 
 from rucio import version
 from rucio.client.client import Client
-from rucio.common.config import config_get_int, config_get
+from rucio.common.config import config_get_int, config_get, config_get_bool
 from rucio.common.exception import (RucioException, RSEWriteBlocked, DataIdentifierAlreadyExists, RSEOperationNotSupported,
                                     DataIdentifierNotFound, NoFilesUploaded, NotAllFilesUploaded, FileReplicaAlreadyExists,
                                     ResourceTemporaryUnavailable, ServiceUnavailable, InputValidationError, RSEChecksumUnavailable,
                                     ScopeNotFound)
 from rucio.common.utils import (adler32, detect_client_location, execute, generate_uuid, make_valid_did, md5, send_trace,
-                                retry, GLOBALLY_SUPPORTED_CHECKSUMS)
+                                retry, bittorrent_v2_merkle_sha256, GLOBALLY_SUPPORTED_CHECKSUMS)
 from rucio.rse import rsemanager as rsemgr
 
 
@@ -336,6 +337,18 @@ class UploadClient:
             raise NotAllFilesUploaded()
         return 0
 
+    def _add_bittorrent_meta(self, file, logger):
+        if not config_get_bool('client', 'register_bittorrent_meta', default=False):
+            return
+
+        pieces_root, pieces_layers, piece_length = bittorrent_v2_merkle_sha256(os.path.join(file['dirname'], file['basename']))
+        bittorrent_meta = {
+            'bittorrent_pieces_root': base64.b64encode(pieces_root).decode(),
+            'bittorrent_pieces_layers': base64.b64encode(pieces_layers).decode(),
+            'bittorrent_piece_length': piece_length,
+        }
+        self.client.set_metadata_bulk(scope=file['did_scope'], name=file['did_name'], meta=bittorrent_meta)
+
     def _register_file(self, file, registered_dataset_dids, ignore_availability=False, activity=None):
         """
         Registers the given file in Rucio. Creates a dataset if
@@ -410,6 +423,7 @@ class UploadClient:
         except DataIdentifierNotFound:
             logger(logging.DEBUG, 'File DID does not exist')
             self.client.add_replicas(rse=rse, files=[replica_for_api])
+            self._add_bittorrent_meta(file=file, logger=logger)
             logger(logging.INFO, 'Successfully added replica in Rucio catalogue at %s' % rse)
             if not dataset_did_str:
                 # only need to add rules for files if no dataset is given

{rucio-33.6.1 → rucio-34.0.0rc2}/lib/rucio/common/constants.py

@@ -48,7 +48,7 @@ if config_get_bool('transfers', 'srm_https_compatibility', raise_exception=False
     SCHEME_MAP['srm'].append('davs')
     SCHEME_MAP['davs'].append('srm')
 
-SUPPORTED_PROTOCOLS = ['gsiftp', 'srm', 'root', 'davs', 'http', 'https', 'file', 'storm', 'srm+https', 'scp', 'rsync', 'rclone']
+SUPPORTED_PROTOCOLS = ['gsiftp', 'srm', 'root', 'davs', 'http', 'https', 'file', 'storm', 'srm+https', 'scp', 'rsync', 'rclone', 'magnet']
 
 FTS_STATE = namedtuple('FTS_STATE', ['SUBMITTED', 'READY', 'ACTIVE', 'FAILED', 'FINISHED', 'FINISHEDDIRTY', 'NOT_USED',
                                      'CANCELED'])('SUBMITTED', 'READY', 'ACTIVE', 'FAILED', 'FINISHED', 'FINISHEDDIRTY',

{rucio-33.6.1/lib/rucio/core → rucio-34.0.0rc2/lib/rucio/common}/plugins.py

@@ -20,8 +20,6 @@ from typing import Any, Callable, Dict, TypeVar, Type
 
 from rucio.common import config
 from rucio.common.exception import InvalidAlgorithmName
-from rucio.common.utils import check_policy_package_version
-from rucio.core.vo import list_vos
 
 
 PolicyPackageAlgorithmsT = TypeVar('PolicyPackageAlgorithmsT', bound='PolicyPackageAlgorithms')
@@ -114,6 +112,7 @@ class PolicyPackageAlgorithms():
                 cls._try_importing_policy(vo)
             # on server, list all VOs and register their algorithms
             else:
+                from rucio.core.vo import list_vos
                 # policy package per VO
                 vos = list_vos()
                 for vo in vos:
@@ -122,6 +121,9 @@ class PolicyPackageAlgorithms():
     @classmethod
     def _try_importing_policy(cls: Type[PolicyPackageAlgorithmsT], vo: str = "") -> None:
         try:
+            # import from utils here to avoid circular import
+            from rucio.common.utils import check_policy_package_version
+
             env_name = 'RUCIO_POLICY_PACKAGE' + ('' if not vo else '_' + vo.upper())
             package = getattr(os.environ, env_name, "")
             if not package:

{rucio-33.6.1 → rucio-34.0.0rc2}/lib/rucio/common/types.py

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import Any, Optional, TypedDict, Union
+from typing import Any, Callable, Literal, Optional, TypedDict, Union
 
 
 class InternalType(object):
@@ -104,6 +104,9 @@ class InternalScope(InternalType):
         super(InternalScope, self).__init__(value=scope, vo=vo, fromExternal=fromExternal)
 
 
+LoggerFunction = Callable[..., Any]
+
+
 class RSEDomainLANDict(TypedDict):
     read: Optional[int]
     write: Optional[int]
@@ -156,3 +159,18 @@ class RSESettingsDict(TypedDict):
     deterministic: bool
     domain: list[str]
     protocols: list[RSEProtocolDict]
+
+
+class RuleDict(TypedDict):
+    account: InternalAccount
+    copies: int
+    rse_expression: str
+    grouping: Literal['ALL', 'DATASET', 'NONE']
+    weight: Optional[str]
+    lifetime: Optional[int]
+    locked: bool
+    subscription_id: Optional[str]
+    source_replica_expression: Optional[str]
+    activity: str
+    notify: Optional[Literal['Y', 'N', 'C', 'P']]
+    purge_replicas: bool