rucio 35.2.1__tar.gz → 35.3.0__tar.gz

{rucio-35.2.1 → rucio-35.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: rucio
-Version: 35.2.1
+Version: 35.3.0
 Summary: Rucio Package
 Home-page: https://rucio.cern.ch/
 Author: Rucio

{rucio-35.2.1 → rucio-35.3.0}/lib/rucio/client/baseclient.py

@@ -171,6 +171,9 @@ class BaseClient:
                 except (NoOptionError, NoSectionError):
                     self.logger.debug('No ca_cert found in configuration. Falling back to Mozilla default CA bundle (certifi).')
                     self.ca_cert = True
+                except ConfigNotFound:
+                    self.logger.debug('No configuration found. Falling back to Mozilla default CA bundle (certifi).')
+                    self.ca_cert = True
 
         if account is None:
             self.logger.debug('No account passed. Trying to get it from the RUCIO_ACCOUNT environment variable or the config file.')
@@ -195,6 +198,9 @@ class BaseClient:
                 except (NoOptionError, NoSectionError):
                     self.logger.debug('No VO found. Using default VO.')
                     self.vo = 'def'
+                except ConfigNotFound:
+                    self.logger.debug('No configuration found. Using default VO.')
+                    self.vo = 'def'
 
         self.auth_token_file_path, self.token_exp_epoch_file, self.token_file, self.token_path = self._get_auth_tokens()
         self.__authenticate()

{rucio-35.2.1 → rucio-35.3.0}/lib/rucio/common/exception.py

@@ -962,11 +962,11 @@ class NoDistance(RucioException):
 
 class PolicyPackageNotFound(RucioException):
     """
-    The policy package specified in the config file cannot be loaded.
+    The policy package specified in the config file was not found
     """
     def __init__(self, *args, **kwargs):
         super(PolicyPackageNotFound, self).__init__(*args, **kwargs)
-        self._message = 'The specified policy package cannot be loaded'
+        self._message = 'The specified policy package was not found'
         self.error_code = 93
 
 
@@ -1099,3 +1099,23 @@ class SortingAlgorithmNotSupported(RucioException):
         super(SortingAlgorithmNotSupported, self).__init__(*args, **kwargs)
         self._message = 'Sorting algorithm is not supported.'
         self.error_code = 106
+
+
+class ErrorLoadingPolicyPackage(RucioException):
+    """
+    An error occurred while loading the policy package.
+    """
+    def __init__(self, *args, **kwargs):
+        super(ErrorLoadingPolicyPackage, self).__init__(*args, **kwargs)
+        self._message = 'An error occurred while loading the specified policy package'
+        self.error_code = 107
+
+
+class TraceValidationSchemaNotFound(RucioException):
+    """
+    Trace validation schema not found.
+    """
+    def __init__(self, *args, **kwargs):
+        super(TraceValidationSchemaNotFound, self).__init__(*args, **kwargs)
+        self._message = 'Trace validation schema not found.'
+        self.error_code = 108

{rucio-35.2.1 → rucio-35.3.0}/lib/rucio/common/schema/__init__.py

@@ -60,8 +60,10 @@ if not multivo:
 
     try:
         module = importlib.import_module(POLICY)
-    except ImportError:
+    except ModuleNotFoundError:
         raise exception.PolicyPackageNotFound('Module ' + POLICY + ' not found')
+    except ImportError:
+        raise exception.ErrorLoadingPolicyPackage('An error occurred while loading module ' + POLICY)
 
     schema_modules["def"] = module
     scope_name_regexps.append(module.SCOPE_NAME_REGEXP)
@@ -90,8 +92,10 @@ def load_schema_for_vo(vo: str) -> None:
 
     try:
         module = importlib.import_module(POLICY)
-    except ImportError:
+    except ModuleNotFoundError:
         raise exception.PolicyPackageNotFound('Module ' + POLICY + ' not found')
+    except ImportError:
+        raise exception.ErrorLoadingPolicyPackage('An error occurred while loading module ' + POLICY)
 
     schema_modules[vo] = module
 

{rucio-35.2.1 → rucio-35.3.0}/lib/rucio/common/types.py

@@ -308,6 +308,10 @@ class TraceDict(TraceBaseDict):
     transferEnd: float
 
 
+class TraceSchemaDict(TypedDict):
+    eventType: str
+
+
 class FileToUploadDict(TypedDict):
     path: PathTypeAlias
     rse: str

{rucio-35.2.1 → rucio-35.3.0}/lib/rucio/core/did.py

@@ -927,12 +927,12 @@ def delete_dids(
 
     # Prepare the common part of the query for updating bad replicas if they exist
     bad_replica_stmt = update(
-        models.BadReplicas
+        models.BadReplica
     ).where(
-        models.BadReplicas.state == BadFilesStatus.BAD
+        models.BadReplica.state == BadFilesStatus.BAD
     ).values({
-        models.BadReplicas.state: BadFilesStatus.DELETED,
-        models.BadReplicas.updated_at: datetime.utcnow(),
+        models.BadReplica.state: BadFilesStatus.DELETED,
+        models.BadReplica.updated_at: datetime.utcnow(),
     }).execution_options(
         synchronize_session=False
     )
@@ -956,8 +956,8 @@ def delete_dids(
             exists(
                 select(1)
             ).where(
-                and_(models.BadReplicas.scope == temp_table.scope,
-                     models.BadReplicas.name == temp_table.name)
+                and_(models.BadReplica.scope == temp_table.scope,
+                     models.BadReplica.name == temp_table.name)
             )
         )
         session.execute(stmt)
@@ -1002,8 +1002,8 @@ def delete_dids(
             exists(
                 select(1)
             ).where(
-                and_(models.BadReplicas.scope == resolved_files_temp_table.scope,
-                     models.BadReplicas.name == resolved_files_temp_table.name)
+                and_(models.BadReplica.scope == resolved_files_temp_table.scope,
+                     models.BadReplica.name == resolved_files_temp_table.name)
             )
         )
         session.execute(stmt)
@@ -1085,13 +1085,13 @@ def delete_dids(
 
         with METRICS.timer('delete_dids.dids_followed'):
             stmt = delete(
-                models.DidsFollowed
+                models.DidFollowed
             ).where(
                 exists(
                     select(1)
                 ).where(
-                    and_(models.DidsFollowed.scope == temp_table.scope,
-                         models.DidsFollowed.name == temp_table.name)
+                    and_(models.DidFollowed.scope == temp_table.scope,
+                         models.DidFollowed.name == temp_table.name)
                 )
             ).execution_options(
                 synchronize_session=False
@@ -2655,8 +2655,8 @@ def add_dids_to_followed(
             )
             did = session.execute(stmt).scalar_one()
             # Add the queried to the followed table.
-            new_did_followed = models.DidsFollowed(scope=did.scope, name=did.name, account=account,
-                                                   did_type=did.did_type)
+            new_did_followed = models.DidFollowed(scope=did.scope, name=did.name, account=account,
+                                                  did_type=did.did_type)
 
             new_did_followed.save(session=session, flush=False)
 
@@ -2681,10 +2681,10 @@ def get_users_following_did(
     """
     try:
         stmt = select(
-            models.DidsFollowed
+            models.DidFollowed
         ).where(
-            and_(models.DidsFollowed.scope == scope,
-                 models.DidsFollowed.name == name)
+            and_(models.DidFollowed.scope == scope,
+                 models.DidFollowed.name == name)
         )
         for user in session.execute(stmt).scalars().all():
             # Return a dictionary of users to be rendered as json.
@@ -2731,11 +2731,11 @@ def remove_dids_from_followed(
     try:
         for did in dids:
             stmt = delete(
-                models.DidsFollowed
+                models.DidFollowed
             ).where(
-                and_(models.DidsFollowed.scope == did['scope'],
-                     models.DidsFollowed.name == did['name'],
-                     models.DidsFollowed.account == account)
+                and_(models.DidFollowed.scope == did['scope'],
+                     models.DidFollowed.name == did['name'],
+                     models.DidFollowed.account == account)
             ).execution_options(
                 synchronize_session=False
             )
@@ -2754,7 +2754,7 @@ def trigger_event(
     session: "Session"
 ) -> None:
     """
-    Records changes occurring in the did to the FollowEvents table
+    Records changes occurring in the did to the FollowEvent table
 
     :param scope: The scope name.
     :param name: The data identifier name.
@@ -2764,15 +2764,15 @@ def trigger_event(
     """
     try:
         stmt = select(
-            models.DidsFollowed
+            models.DidFollowed
         ).where(
-            and_(models.DidsFollowed.scope == scope,
-                 models.DidsFollowed.name == name)
+            and_(models.DidFollowed.scope == scope,
+                 models.DidFollowed.name == name)
         )
         for did in session.execute(stmt).scalars().all():
             # Create a new event using the specified parameters.
-            new_event = models.FollowEvents(scope=scope, name=name, account=did.account,
-                                            did_type=did.did_type, event_type=event_type, payload=payload)
+            new_event = models.FollowEvent(scope=scope, name=name, account=did.account,
+                                           did_type=did.did_type, event_type=event_type, payload=payload)
             new_event.save(session=session, flush=False)
 
         session.flush()
@@ -2792,11 +2792,11 @@ def create_reports(
 
     :param session: The database session in use.
     """
-    # Query the FollowEvents table
+    # Query the FollowEvent table
     stmt = select(
-        models.FollowEvents
+        models.FollowEvent
     ).order_by(
-        models.FollowEvents.created_at
+        models.FollowEvent.created_at
     )
 
     # Use heartbeat mechanism to select a chunk of events based on the hashed account
@@ -2821,11 +2821,11 @@ def create_reports(
                 account = event.account
                 # Clean up the event after creating the report
                 stmt = delete(
-                    models.FollowEvents
+                    models.FollowEvent
                 ).where(
-                    and_(models.FollowEvents.scope == event.scope,
-                         models.FollowEvents.name == event.name,
-                         models.FollowEvents.account == event.account)
+                    and_(models.FollowEvent.scope == event.scope,
+                         models.FollowEvent.name == event.name,
+                         models.FollowEvent.account == event.account)
                 ).execution_options(
                     synchronize_session=False
                 )

{rucio-35.2.1 → rucio-35.3.0}/lib/rucio/core/did_meta_plugins/__init__.py

@@ -61,8 +61,10 @@ for meta_module_path in METADATA_PLUGIN_MODULE_PATHS:
         base_class = meta_module_path.split(".")[-1]
         metadata_plugin_module = getattr(importlib.import_module(base_module), base_class)()
         METADATA_PLUGIN_MODULES.append(metadata_plugin_module)
-    except ImportError:
+    except ModuleNotFoundError:
         raise exception.PolicyPackageNotFound('Module ' + meta_module_path + ' not found')
+    except ImportError:
+        raise exception.ErrorLoadingPolicyPackage('An error occurred while loading module ' + meta_module_path)
 
 # Set restricted character set for metadata in form character: reason
 #

{rucio-35.2.1 → rucio-35.3.0}/lib/rucio/core/heartbeat.py

@@ -20,7 +20,7 @@ from sqlalchemy import and_, delete, func, select, update
 
 from rucio.common.exception import DatabaseException
 from rucio.common.utils import pid_exists
-from rucio.db.sqla.models import Heartbeats
+from rucio.db.sqla.models import Heartbeat
 from rucio.db.sqla.session import read_session, transactional_session
 
 if TYPE_CHECKING:
@@ -93,36 +93,36 @@ def _sanity_check(
     :param session: The database session in use.
     """
     base_stmt = select(
-        Heartbeats.pid
+        Heartbeat.pid
     ).distinct(
     ).where(
-        Heartbeats.hostname == hostname
+        Heartbeat.hostname == hostname
     )
     if executable:
         if not hash_executable:
             hash_executable = calc_hash(executable)
 
         stmt = base_stmt.where(
-            Heartbeats.executable == hash_executable
+            Heartbeat.executable == hash_executable
         )
         for pid in session.execute(stmt).scalars().all():
             if not pid_exists(pid):
                 stmt = delete(
-                    Heartbeats
+                    Heartbeat
                 ).where(
-                    and_(Heartbeats.executable == hash_executable,
-                         Heartbeats.hostname == hostname,
-                         Heartbeats.pid == pid)
+                    and_(Heartbeat.executable == hash_executable,
+                         Heartbeat.hostname == hostname,
+                         Heartbeat.pid == pid)
                 )
                 session.execute(stmt)
     else:
         for pid in session.execute(base_stmt).scalars().all():
             if not pid_exists(pid):
                 stmt = delete(
-                    Heartbeats
+                    Heartbeat
                 ).where(
-                    and_(Heartbeats.hostname == hostname,
-                         Heartbeats.pid == pid)
+                    and_(Heartbeat.hostname == hostname,
+                         Heartbeat.pid == pid)
                 )
                 session.execute(stmt)
 
@@ -172,45 +172,45 @@ def live(
 
     # upsert the heartbeat
     stmt = update(
-        Heartbeats
+        Heartbeat
     ).where(
-        and_(Heartbeats.executable == hash_executable,
-             Heartbeats.hostname == hostname,
-             Heartbeats.pid == pid,
-             Heartbeats.thread_id == thread_id)
+        and_(Heartbeat.executable == hash_executable,
+             Heartbeat.hostname == hostname,
+             Heartbeat.pid == pid,
+             Heartbeat.thread_id == thread_id)
     ).values({
-        Heartbeats.updated_at: datetime.datetime.utcnow(),
-        Heartbeats.payload: payload
+        Heartbeat.updated_at: datetime.datetime.utcnow(),
+        Heartbeat.payload: payload
     })
     if not session.execute(stmt).rowcount:
-        Heartbeats(executable=hash_executable,
-                   readable=executable[:Heartbeats.readable.property.columns[0].type.length],
-                   hostname=hostname,
-                   pid=pid,
-                   thread_id=thread_id,
-                   thread_name=thread_name,
-                   payload=payload).save(session=session)
+        Heartbeat(executable=hash_executable,
+                  readable=executable[:Heartbeat.readable.property.columns[0].type.length],
+                  hostname=hostname,
+                  pid=pid,
+                  thread_id=thread_id,
+                  thread_name=thread_name,
+                  payload=payload).save(session=session)
 
     # assign thread identifier
     stmt = select(
-        Heartbeats.hostname,
-        Heartbeats.pid,
-        Heartbeats.thread_id
+        Heartbeat.hostname,
+        Heartbeat.pid,
+        Heartbeat.thread_id
     ).with_hint(
-        Heartbeats,
+        Heartbeat,
         'INDEX(HEARTBEATS HEARTBEATS_PK)',
         'oracle'
     ).where(
-        and_(Heartbeats.executable == hash_executable,
-             Heartbeats.updated_at >= datetime.datetime.utcnow() - datetime.timedelta(seconds=older_than))
+        and_(Heartbeat.executable == hash_executable,
+             Heartbeat.updated_at >= datetime.datetime.utcnow() - datetime.timedelta(seconds=older_than))
     ).group_by(
-        Heartbeats.hostname,
-        Heartbeats.pid,
-        Heartbeats.thread_id
+        Heartbeat.hostname,
+        Heartbeat.pid,
+        Heartbeat.thread_id
     ).order_by(
-        Heartbeats.hostname,
-        Heartbeats.pid,
-        Heartbeats.thread_id
+        Heartbeat.hostname,
+        Heartbeat.pid,
+        Heartbeat.thread_id
     )
     result = session.execute(stmt).all()
 
@@ -252,17 +252,17 @@ def die(
         hash_executable = calc_hash(executable)
 
     stmt = delete(
-        Heartbeats
+        Heartbeat
     ).where(
-        and_(Heartbeats.executable == hash_executable,
-             Heartbeats.hostname == hostname,
-             Heartbeats.pid == pid,
-             Heartbeats.thread_id == thread.ident)
+        and_(Heartbeat.executable == hash_executable,
+             Heartbeat.hostname == hostname,
+             Heartbeat.pid == pid,
+             Heartbeat.thread_id == thread.ident)
     )
 
     if older_than:
         stmt = stmt.where(
-            Heartbeats.updated_at < datetime.datetime.utcnow() - datetime.timedelta(seconds=older_than)
+            Heartbeat.updated_at < datetime.datetime.utcnow() - datetime.timedelta(seconds=older_than)
         )
     session.execute(stmt)
 
@@ -277,12 +277,12 @@ def cardiac_arrest(older_than: Optional[int] = None, *, session: "Session") -> N
     """
 
     stmt = delete(
-        Heartbeats
+        Heartbeat
     )
 
     if older_than:
         stmt = stmt.where(
-            Heartbeats.updated_at < datetime.datetime.utcnow() - datetime.timedelta(seconds=older_than)
+            Heartbeat.updated_at < datetime.datetime.utcnow() - datetime.timedelta(seconds=older_than)
         )
     session.execute(stmt)
 
@@ -298,17 +298,17 @@ def list_heartbeats(*, session: "Session") -> list["HeartbeatDict"]:
     """
 
     stmt = select(
-        Heartbeats.readable,
-        Heartbeats.hostname,
-        Heartbeats.pid,
-        Heartbeats.thread_name,
-        Heartbeats.updated_at,
-        Heartbeats.created_at,
-        Heartbeats.payload
+        Heartbeat.readable,
+        Heartbeat.hostname,
+        Heartbeat.pid,
+        Heartbeat.thread_name,
+        Heartbeat.updated_at,
+        Heartbeat.created_at,
+        Heartbeat.payload
     ).order_by(
-        Heartbeats.readable,
-        Heartbeats.hostname,
-        Heartbeats.thread_name
+        Heartbeat.readable,
+        Heartbeat.hostname,
+        Heartbeat.thread_name
     )
 
     result = session.execute(stmt).all()
@@ -340,15 +340,15 @@ def list_payload_counts(
     if not hash_executable:
         hash_executable = calc_hash(executable)
     stmt = select(
-        Heartbeats.payload,
-        func.count(Heartbeats.payload)
+        Heartbeat.payload,
+        func.count(Heartbeat.payload)
     ).where(
-        and_(Heartbeats.executable == hash_executable,
-             Heartbeats.updated_at >= datetime.datetime.utcnow() - datetime.timedelta(seconds=older_than))
+        and_(Heartbeat.executable == hash_executable,
+             Heartbeat.updated_at >= datetime.datetime.utcnow() - datetime.timedelta(seconds=older_than))
     ).group_by(
-        Heartbeats.payload
+        Heartbeat.payload
     ).order_by(
-        Heartbeats.payload
+        Heartbeat.payload
     )
 
     return dict((payload, count) for payload, count in session.execute(stmt).all() if payload)

{rucio-35.2.1 → rucio-35.3.0}/lib/rucio/core/lifetime_exception.py

@@ -56,9 +56,9 @@ def list_exceptions(
 
     state_clause = []
     if states:
-        state_clause = [models.LifetimeExceptions.state == state for state in states]
+        state_clause = [models.LifetimeException.state == state for state in states]
 
-    query = select(models.LifetimeExceptions)
+    query = select(models.LifetimeException)
     if state_clause != []:
         query = query.where(or_(*state_clause))
     if exception_id:
@@ -201,8 +201,8 @@ def __add_exception(
                 did_type = DIDType[did['did_type']]
             else:
                 did_type = did['did_type']
-        new_exception = models.LifetimeExceptions(id=exception_id, scope=did['scope'], name=did['name'], did_type=did_type,
-                                                  account=account, pattern=pattern, comments=reason, state=LifetimeExceptionsState.WAITING, expires_at=lifetime)
+        new_exception = models.LifetimeException(id=exception_id, scope=did['scope'], name=did['name'], did_type=did_type,
+                                                 account=account, pattern=pattern, comments=reason, state=LifetimeExceptionsState.WAITING, expires_at=lifetime)
         if len(text) < 3000:
             text += '%s %s %s\n' % (str(did_type), did['scope'], did['name'])
         else:
@@ -250,9 +250,9 @@ def update_exception(
         raise UnsupportedOperation
 
     query = update(
-        models.LifetimeExceptions
+        models.LifetimeException
     ).where(
-        models.LifetimeExceptions.id == exception_id
+        models.LifetimeException.id == exception_id
     ).values(
         state=state,
         updated_at=datetime.utcnow()

{rucio-35.2.1 → rucio-35.3.0}/lib/rucio/core/meta_conventions.py

@@ -121,7 +121,7 @@ def add_value(key: str, value: str, *, session: "Session") -> None:
     :raises KeyNotFound: Key not in metadata conventions table
     :raises InvalidValueForKey: Value conflicts with rse expression for key values or does not have the correct type
     """
-    new_value = models.DIDMetaConventionsConstraints(key=key, value=value)
+    new_value = models.DIDMetaConventionsConstraint(key=key, value=value)
     try:
         new_value.save(session=session)
     except IntegrityError as error:
@@ -170,9 +170,9 @@ def list_values(key: str, *, session: "Session") -> list[str]:
     :returns: A list containing all values.
     """
     statement = select(
-        models.DIDMetaConventionsConstraints.value
+        models.DIDMetaConventionsConstraint.value
     ).where(
-        models.DIDMetaConventionsConstraints.key == key
+        models.DIDMetaConventionsConstraint.key == key
     )
     return list(session.execute(statement).scalars().all())
 
@@ -193,10 +193,10 @@ def validate_meta(meta: dict, did_type: DIDType, *, session: "Session") -> None:
     if did_type == DIDType.DATASET and key in meta:
         try:
             statement = select(
-                models.DIDMetaConventionsConstraints.value
+                models.DIDMetaConventionsConstraint.value
             ).where(
-                and_(models.DIDMetaConventionsConstraints.value == meta[key],
-                     models.DIDMetaConventionsConstraints.key == key)
+                and_(models.DIDMetaConventionsConstraint.value == meta[key],
+                     models.DIDMetaConventionsConstraint.key == key)
             )
             session.execute(statement).one()
         except NoResultFound:

{rucio-35.2.1 → rucio-35.3.0}/lib/rucio/core/nongrid_trace.py

@@ -16,6 +16,7 @@ import json
 import logging.handlers
 import random
 import socket
+from typing import TYPE_CHECKING, Any, Union, overload
 
 import stomp
 
@@ -23,6 +24,9 @@ from rucio.common.config import config_get, config_get_int
 from rucio.common.logging import rucio_log_formatter
 from rucio.core.monitor import MetricManager
 
+if TYPE_CHECKING:
+    from datetime import datetime
+
 METRICS = MetricManager(module=__name__)
 
 CONFIG_COMMON_LOGLEVEL = getattr(logging, config_get('common', 'loglevel', raise_exception=False, default='DEBUG').upper())
@@ -79,14 +83,23 @@ for broker in BROKERS_RESOLVED:
     CONNS.append(stomp.Connection(host_and_ports=[(broker, PORT)], vhost=VHOST, reconnect_attempts_max=3))
 
 
-def date_handler(obj):
-    '''
-    '''
+@overload
+def date_handler(obj: "datetime") -> str:
+    ...
+
+
+@overload
+def date_handler(obj: object) -> object:
+    ...
+
+
+def date_handler(obj: Any) -> Union[str, object]:
+    """ Format dates to ISO format. """
     return obj.isoformat() if hasattr(obj, 'isoformat') else obj
 
 
 @METRICS.count_it
-def trace(payload):
+def trace(payload: dict[str, Any]) -> None:
     """
     Write a trace to the buffer log file and send it to active mq.
 

{rucio-35.2.1 → rucio-35.3.0}/lib/rucio/core/permission/__init__.py

@@ -68,8 +68,10 @@ if not multivo:
 
     try:
         module = importlib.import_module(POLICY)
-    except ImportError:
+    except ModuleNotFoundError:
         raise exception.PolicyPackageNotFound('Module ' + POLICY + ' not found')
+    except ImportError:
+        raise exception.ErrorLoadingPolicyPackage('An error occurred while loading module ' + POLICY)
 
     permission_modules["def"] = module
 
@@ -97,8 +99,10 @@ def load_permission_for_vo(vo: str) -> None:
 
     try:
         module = importlib.import_module(POLICY)
-    except ImportError:
+    except ModuleNotFoundError:
         raise exception.PolicyPackageNotFound('Module ' + POLICY + ' not found')
+    except ImportError:
+        raise exception.ErrorLoadingPolicyPackage('An error occurred while loading module ' + POLICY)
 
     permission_modules[vo] = module