rucio 35.1.1__tar.gz → 35.2.1__tar.gz

{rucio-35.1.1 → rucio-35.2.1}/MANIFEST.in

@@ -2,7 +2,7 @@ include MANIFEST.in pylintrc
 include AUTHORS
 include LICENSE
 include ChangeLog
-include README.rst
+include README.md
 include setuputil.py
 include setup.py
 include setup.cfg

{rucio-35.1.1 → rucio-35.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: rucio
-Version: 35.1.1
+Version: 35.2.1
 Summary: Rucio Package
 Home-page: https://rucio.cern.ch/
 Author: Rucio

{rucio-35.1.1 → rucio-35.2.1}/lib/rucio/client/baseclient.py

@@ -137,6 +137,7 @@ class BaseClient:
             self.logger.debug('No trace_host passed. Using rucio_host instead')
 
         self.list_hosts = [self.host]
+        self.account = account
         self.ca_cert = ca_cert
         self.auth_token = ""
         self.headers = {}
@@ -171,9 +172,7 @@ class BaseClient:
                     self.logger.debug('No ca_cert found in configuration. Falling back to Mozilla default CA bundle (certifi).')
                     self.ca_cert = True
 
-        if account is not None:
-            self.account = account
-        else:
+        if account is None:
             self.logger.debug('No account passed. Trying to get it from the RUCIO_ACCOUNT environment variable or the config file.')
             try:
                 self.account = environ['RUCIO_ACCOUNT']

{rucio-35.1.1 → rucio-35.2.1}/lib/rucio/common/dumper/consistency.py

@@ -163,9 +163,9 @@ def _try_to_advance(it, default=None):
     return el.strip()
 
 
-def min3(*values):
+def min_value(*values):
     '''
-    Minimum between the 3 values ignoring None
+    Minimum between the input values, ignoring None
     '''
     values = [value for value in values if value is not None]
     if len(values) == 0:
@@ -201,7 +201,7 @@ def compare3(it0, it1, it2):
         path0, status0 = split_if_not_none(v0)
         path2, status2 = split_if_not_none(v2)
 
-        vmin = min3(path0, v1, path2)
+        vmin = min_value(path0, v1, path2)
         in0 = in1 = in2 = False
         in0_status = in2_status = None
 

{rucio-35.1.1 → rucio-35.2.1}/lib/rucio/common/types.py

@@ -27,7 +27,7 @@ else:
 
 if TYPE_CHECKING:
     from rucio.common.constants import SUPPORTED_PROTOCOLS_LITERAL
-    from rucio.db.sqla.constants import AccountType, IdentityType
+    from rucio.db.sqla.constants import AccountType, IdentityType, RequestState, RequestType
 
 
 class InternalType:
@@ -343,3 +343,57 @@ class FileToUploadWithCollectedInfoDict(FileToUploadDict):
 class FileToUploadWithCollectedAndDatasetInfoDict(FileToUploadWithCollectedInfoDict):
     dataset_scope: str
     dataset_name: str
+
+
+class RequestGatewayDict(TypedDict):
+    """
+    Request dict expected as input to gateway
+    """
+    scope: str
+    name: str
+    account: Optional[str]
+    dest_rse_id: str
+    request_type: "RequestType"
+    attributes: "RequestAttributesDict"
+
+
+class RequestDict(TypedDict):
+    """
+    Requested dict used in core
+    """
+    id: str
+    request_id: str
+    scope: InternalScope
+    name: str
+    source_rse_id: str
+    dest_rse_id: str
+    dest_url: str
+    state: "RequestState"
+    account: NotRequired[InternalAccount]
+    rule_id: str
+    adler32: str
+    bytes: int
+    err_msg: str
+    sources: list[dict[str, Any]]
+    request_type: "RequestType"
+    retry_count: Optional[int]
+    previous_attempt_id: str
+    external_host: str
+    external_id: str
+    transfertool: str
+    attributes: "RequestAttributesDict"
+
+
+class RequestAttributesDict(TypedDict):
+    activity: str
+    bytes: int
+    md5: str
+    adler32: str
+    is_intermediate_hop: bool
+
+
+class FilterDict(TypedDict):
+    rule_id: str
+    request_id: str
+    older_than: datetime
+    activities: Union[list[str], str]

{rucio-35.1.1 → rucio-35.2.1}/lib/rucio/core/did_meta_plugins/did_column_meta.py

@@ -219,7 +219,7 @@ class DidColumnMeta(DidMetaPlugin):
 
         # instantiate fe and create sqla query
         fe = FilterEngine(filters, model_class=models.DataIdentifier)
-        query = fe.create_sqla_query(
+        stmt = fe.create_sqla_query(
             additional_model_attributes=[
                 models.DataIdentifier.scope,
                 models.DataIdentifier.name,
@@ -232,16 +232,22 @@ class DidColumnMeta(DidMetaPlugin):
             ],
             session=session
         )
-        query.with_hint(models.DataIdentifier, 'NO_EXPAND', 'oracle')
+        stmt = stmt.with_hint(
+            models.DataIdentifier,
+            'NO_EXPAND',
+            'oracle'
+        )
 
         if limit:
-            query = query.limit(limit)
+            stmt = stmt.limit(
+                limit
+            )
         if recursive:
             from rucio.core.did import list_content
 
             # Get attached DIDs and save in list because query has to be finished before starting a new one in the recursion
             collections_content = []
-            for did in query.yield_per(100):
+            for did in session.execute(stmt).yield_per(100):
                 if (did.did_type == DIDType.CONTAINER or did.did_type == DIDType.DATASET):
                     collections_content += [d for d in list_content(scope=did.scope, name=did.name)]
 
@@ -253,7 +259,7 @@ class DidColumnMeta(DidMetaPlugin):
                                              long=long, ignore_dids=ignore_dids, session=session):
                     yield result
 
-        for did in query.yield_per(5):                  # don't unpack this as it makes it dependent on query return order!
+        for did in session.execute(stmt).yield_per(5):                  # don't unpack this as it makes it dependent on query return order!
             if long:
                 did_full = "{}:{}".format(did.scope, did.name)
                 if did_full not in ignore_dids:         # concatenating results of OR clauses may contain duplicate DIDs if query result sets not mutually exclusive.
@@ -261,7 +267,7 @@ class DidColumnMeta(DidMetaPlugin):
                     yield {
                         'scope': did.scope,
                         'name': did.name,
-                        'did_type': str(did.did_type),
+                        'did_type': did.did_type.name,
                         'bytes': did.bytes,
                         'length': did.length
                     }

{rucio-35.1.1 → rucio-35.2.1}/lib/rucio/core/did_meta_plugins/filter_engine.py

@@ -20,8 +20,8 @@ from importlib import import_module
 from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union
 
 import sqlalchemy
-from sqlalchemy import and_, cast, or_
-from sqlalchemy.orm import InstrumentedAttribute, Query
+from sqlalchemy import Select, and_, cast, or_, select
+from sqlalchemy.orm import InstrumentedAttribute
 from sqlalchemy.sql.expression import text
 
 from rucio.common import exception
@@ -434,7 +434,7 @@ class FilterEngine:
         additional_model_attributes: Optional[list[InstrumentedAttribute[Any]]] = None,
         additional_filters: Optional["Iterable[FilterTuple]"] = None,
         json_column: Optional[InstrumentedAttribute] = None
-    ) -> Query:
+    ) -> Select:
         """
         Returns a database query that fully describes the filters.
 
@@ -445,7 +445,7 @@ class FilterEngine:
         :param additional_model_attributes: Additional model attributes to retrieve.
         :param additional_filters: Additional filters to be applied to all clauses.
         :param json_column: Column to be checked if filter key has not been coerced to a model attribute. Only valid if engine instantiated with strict_coerce=False.
-        :returns: A database query.
+        :returns: A SQLAlchemy Select object.
         :raises: FilterEngineGenericError
         """
         additional_model_attributes = additional_model_attributes or []
@@ -525,7 +525,12 @@ class FilterEngine:
 
                 and_expressions.append(expression)
             or_expressions.append(and_(*and_expressions))
-        return session.query(*all_model_attributes).filter(or_(*or_expressions))
+        stmt = select(
+            *all_model_attributes
+        ).where(
+            or_(*or_expressions)
+        )
+        return stmt
 
     def evaluate(self) -> bool:
         """

{rucio-35.1.1 → rucio-35.2.1}/lib/rucio/core/did_meta_plugins/json_meta.py

@@ -175,7 +175,7 @@ class JSONDidMeta(DidMetaPlugin):
         # instantiate fe and create sqla query, note that coercion to a model keyword
         # is not appropriate here as the filter words are stored in a single json column.
         fe = FilterEngine(filters, model_class=models.DidMeta, strict_coerce=False)
-        query = fe.create_sqla_query(
+        stmt = fe.create_sqla_query(
             additional_model_attributes=[
                 models.DidMeta.scope,
                 models.DidMeta.name
@@ -187,13 +187,15 @@ class JSONDidMeta(DidMetaPlugin):
         )
 
         if limit:
-            query = query.limit(limit)
+            stmt = stmt.limit(
+                limit
+            )
         if recursive:
             from rucio.core.did import list_content
 
             # Get attached DIDs and save in list because query has to be finished before starting a new one in the recursion
             collections_content = []
-            for did in query.yield_per(100):
+            for did in session.execute(stmt).yield_per(100):
                 if (did.did_type == DIDType.CONTAINER or did.did_type == DIDType.DATASET):
                     collections_content += [d for d in list_content(scope=did.scope, name=did.name)]
 
@@ -206,7 +208,7 @@ class JSONDidMeta(DidMetaPlugin):
                     yield result
 
         try:
-            for did in query.yield_per(5):                  # don't unpack this as it makes it dependent on query return order!
+            for did in session.execute(stmt).yield_per(5):                  # don't unpack this as it makes it dependent on query return order!
                 if long:
                     did_full = "{}:{}".format(did.scope, did.name)
                     if did_full not in ignore_dids:         # concatenating results of OR clauses may contain duplicate DIDs if query result sets not mutually exclusive.

{rucio-35.1.1 → rucio-35.2.1}/lib/rucio/core/replica.py

@@ -2554,17 +2554,24 @@ def update_replicas_states(replicas, nowait=False, *, session: "Session"):
         elif replica['state'] == ReplicaState.AVAILABLE:
             rucio.core.lock.successful_transfer(scope=replica['scope'], name=replica['name'], rse_id=replica['rse_id'], nowait=nowait, session=session)
             stmt_bad_replicas = select(
+                func.count()
+            ).select_from(
                 models.BadReplicas
             ).where(
                 and_(models.BadReplicas.state == BadFilesStatus.BAD,
                      models.BadReplicas.rse_id == replica['rse_id'],
                      models.BadReplicas.scope == replica['scope'],
-                     models.BadReplicas.name == replica['name']))
-            if session.execute(stmt_bad_replicas).scalar_one_or_none():
+                     models.BadReplicas.name == replica['name'])
+            )
+
+            if session.execute(stmt_bad_replicas).scalar():
                 update_stmt = update(
                     models.BadReplicas
                 ).where(
-                    models.BadReplicas.rse_id == replica['rse_id']
+                    and_(models.BadReplicas.state == BadFilesStatus.BAD,
+                         models.BadReplicas.rse_id == replica['rse_id'],
+                         models.BadReplicas.scope == replica['scope'],
+                         models.BadReplicas.name == replica['name'])
                 ).values({
                     models.BadReplicas.state: BadFilesStatus.RECOVERED,
                     models.BadReplicas.updated_at: datetime.utcnow()