rucio 32.8.6__py3-none-any.whl → 35.8.0__py3-none-any.whl

rucio/core/did_meta_plugins/__init__.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -62,8 +61,10 @@ for meta_module_path in METADATA_PLUGIN_MODULE_PATHS:
         base_class = meta_module_path.split(".")[-1]
         metadata_plugin_module = getattr(importlib.import_module(base_module), base_class)()
         METADATA_PLUGIN_MODULES.append(metadata_plugin_module)
+    except ModuleNotFoundError:
+        raise exception.PolicyPackageNotFound(meta_module_path)
     except ImportError:
-        raise exception.PolicyPackageNotFound('Module ' + meta_module_path + ' not found')
+        raise exception.ErrorLoadingPolicyPackage(meta_module_path)
 
 # Set restricted character set for metadata in form character: reason
 #
@@ -83,8 +84,9 @@ def get_metadata(scope, name, plugin="DID_COLUMN", *, session: "Session"):
     :param scope: The scope of the did.
     :param name: The data identifier name.
     :param plugin: (optional) Filter specific metadata plugins.
-    :returns: List of metadata for did.
-    :raises: NotImplementedError
+    :param session: (optional) The database session in use.
+    :returns: Dictionary containing metadata for did.
+    :raises: UnsupportedMetadataPlugin: If the specified plugin is not enabled/available
     """
     if plugin.lower() == "all":
         all_metadata = {}
@@ -96,7 +98,7 @@ def get_metadata(scope, name, plugin="DID_COLUMN", *, session: "Session"):
         for metadata_plugin in METADATA_PLUGIN_MODULES:
             if metadata_plugin.get_plugin_name().lower() == plugin.lower():
                 return metadata_plugin.get_metadata(scope, name, session=session)
-    raise NotImplementedError('Metadata plugin "%s" is not enabled on the server.' % plugin)
+    raise exception.UnsupportedMetadataPlugin(f'Metadata plugin "{plugin}" is not enabled on the server.')
 
 
 @transactional_session
@@ -216,7 +218,7 @@ def list_dids(scope=None, filters=None, did_type='collection', ignore_case=False
     :returns: List of dids satisfying metadata criteria.
     :raises: InvalidMetadata
     """
-    # backwards compatability for filters as single {}.
+    # backwards compatibility for filters as single {}.
     if isinstance(filters, dict):
         filters = [filters]
 

rucio/core/did_meta_plugins/did_column_meta.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -17,9 +16,8 @@ import operator
 from datetime import datetime, timedelta
 from typing import TYPE_CHECKING
 
-from sqlalchemy import update, inspect
-from sqlalchemy.exc import CompileError, InvalidRequestError
-from sqlalchemy.orm.exc import NoResultFound
+from sqlalchemy import inspect, update
+from sqlalchemy.exc import CompileError, InvalidRequestError, NoResultFound
 from sqlalchemy.sql import func
 from sqlalchemy.sql.expression import true
 
@@ -29,10 +27,11 @@ from rucio.core.did_meta_plugins.did_meta_plugin_interface import DidMetaPlugin
 from rucio.core.did_meta_plugins.filter_engine import FilterEngine
 from rucio.db.sqla import models
 from rucio.db.sqla.constants import DIDType
-from rucio.db.sqla.session import stream_session, read_session, transactional_session
+from rucio.db.sqla.session import read_session, stream_session, transactional_session
 
 if TYPE_CHECKING:
     from typing import Optional
+
     from sqlalchemy.orm import Session
 
 
@@ -198,7 +197,7 @@ class DidColumnMeta(DidMetaPlugin):
             'file': [DIDType.FILE]
         }
 
-        # backwards compatability for filters as single {}.
+        # backwards compatibility for filters as single {}.
         if isinstance(filters, dict):
             filters = [filters]
 
@@ -220,7 +219,7 @@ class DidColumnMeta(DidMetaPlugin):
 
         # instantiate fe and create sqla query
         fe = FilterEngine(filters, model_class=models.DataIdentifier)
-        query = fe.create_sqla_query(
+        stmt = fe.create_sqla_query(
             additional_model_attributes=[
                 models.DataIdentifier.scope,
                 models.DataIdentifier.name,
@@ -233,16 +232,22 @@ class DidColumnMeta(DidMetaPlugin):
             ],
             session=session
         )
-        query.with_hint(models.DataIdentifier, 'NO_EXPAND', 'oracle')
+        stmt = stmt.with_hint(
+            models.DataIdentifier,
+            'NO_EXPAND',
+            'oracle'
+        )
 
         if limit:
-            query = query.limit(limit)
+            stmt = stmt.limit(
+                limit
+            )
         if recursive:
             from rucio.core.did import list_content
 
             # Get attached DIDs and save in list because query has to be finished before starting a new one in the recursion
             collections_content = []
-            for did in query.yield_per(100):
+            for did in session.execute(stmt).yield_per(100):
                 if (did.did_type == DIDType.CONTAINER or did.did_type == DIDType.DATASET):
                     collections_content += [d for d in list_content(scope=did.scope, name=did.name)]
 
@@ -254,7 +259,7 @@ class DidColumnMeta(DidMetaPlugin):
                                              long=long, ignore_dids=ignore_dids, session=session):
                     yield result
 
-        for did in query.yield_per(5):                  # don't unpack this as it makes it dependent on query return order!
+        for did in session.execute(stmt).yield_per(5):                  # don't unpack this as it makes it dependent on query return order!
             if long:
                 did_full = "{}:{}".format(did.scope, did.name)
                 if did_full not in ignore_dids:         # concatenating results of OR clauses may contain duplicate DIDs if query result sets not mutually exclusive.
@@ -262,7 +267,7 @@ class DidColumnMeta(DidMetaPlugin):
                     yield {
                         'scope': did.scope,
                         'name': did.name,
-                        'did_type': str(did.did_type),
+                        'did_type': did.did_type.name,
                         'bytes': did.bytes,
                         'length': did.length
                     }

rucio/core/did_meta_plugins/did_meta_plugin_interface.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,16 +13,20 @@
 # limitations under the License.
 
 from abc import ABCMeta, abstractmethod
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Literal
 
 from rucio.db.sqla.session import transactional_session
 
 if TYPE_CHECKING:
-    from typing import Optional
+    from collections.abc import Iterator
+    from typing import Any, Optional, Union
+
     from sqlalchemy.orm import Session
 
+    from rucio.common.types import InternalScope
+
 
-class DidMetaPlugin(object, metaclass=ABCMeta):
+class DidMetaPlugin(metaclass=ABCMeta):
     """
     Interface for plugins managing metadata of DIDs
     """
@@ -35,7 +38,13 @@ class DidMetaPlugin(object, metaclass=ABCMeta):
         pass
 
     @abstractmethod
-    def get_metadata(self, scope, name, *, session: "Optional[Session]" = None):
+    def get_metadata(
+        self,
+        scope: "InternalScope",
+        name: str,
+        *,
+        session: "Optional[Session]" = None
+    ) -> "Any":
         """
         Get data identifier metadata
 
@@ -46,7 +55,16 @@ class DidMetaPlugin(object, metaclass=ABCMeta):
         pass
 
     @abstractmethod
-    def set_metadata(self, scope, name, key, value, recursive=False, *, session: "Optional[Session]" = None):
+    def set_metadata(
+        self,
+        scope: "InternalScope",
+        name: str,
+        key: str,
+        value: str,
+        recursive: bool = False,
+        *,
+        session: "Optional[Session]" = None
+    ) -> None:
         """
         Add metadata to data identifier.
 
@@ -61,7 +79,15 @@ class DidMetaPlugin(object, metaclass=ABCMeta):
         pass
 
     @transactional_session
-    def set_metadata_bulk(self, scope, name, meta, recursive=False, *, session: "Optional[Session]" = None):
+    def set_metadata_bulk(
+        self,
+        scope: "InternalScope",
+        name: str,
+        meta: dict[str, "Any"],
+        recursive: bool = False,
+        *,
+        session: "Optional[Session]" = None
+    ) -> None:
         """
         Add metadata to data identifier in bulk.
 
@@ -76,7 +102,14 @@ class DidMetaPlugin(object, metaclass=ABCMeta):
             self.set_metadata(scope, name, key, value, recursive=recursive, session=session)
 
     @abstractmethod
-    def delete_metadata(self, scope, name, key, *, session: "Session" = None):
+    def delete_metadata(
+        self,
+        scope: "InternalScope",
+        name: str,
+        key: str,
+        *,
+        session: "Optional[Session]" = None
+    ) -> None:
         """
         Deletes the metadata stored for the given key.
 
@@ -88,8 +121,19 @@ class DidMetaPlugin(object, metaclass=ABCMeta):
         pass
 
     @abstractmethod
-    def list_dids(self, scope, filters, did_type='collection', ignore_case=False, limit=None,
-                  offset=None, long=False, recursive=False, *, session: "Optional[Session]" = None):
+    def list_dids(
+        self,
+        scope: "InternalScope",
+        filters: dict[str, "Any"],
+        did_type: Literal['all', 'collection', 'dataset', 'container', 'file'] = 'collection',
+        ignore_case: bool = False,
+        limit: "Optional[int]" = None,
+        offset: "Optional[int]" = None,
+        long: bool = False,
+        recursive: bool = False,
+        *,
+        session: "Optional[Session]" = None
+    ) -> "Iterator[Union[str, dict[str, Any]]]":
         """
         Search data identifiers
 
@@ -106,7 +150,12 @@ class DidMetaPlugin(object, metaclass=ABCMeta):
         pass
 
     @abstractmethod
-    def manages_key(self, key, *, session: "Optional[Session]" = None):
+    def manages_key(
+        self,
+        key: str,
+        *,
+        session: "Optional[Session]" = None
+    ) -> bool:
         """
         Returns whether key is managed by this plugin or not.
         :param key: Key of the metadata.

rucio/core/did_meta_plugins/filter_engine.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -16,12 +15,13 @@
 import ast
 import fnmatch
 import operator
-from datetime import datetime, timedelta, date
+from datetime import date, datetime, timedelta
 from importlib import import_module
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union
 
 import sqlalchemy
-from sqlalchemy import cast, or_, and_
+from sqlalchemy import Select, and_, cast, or_, select
+from sqlalchemy.orm import InstrumentedAttribute
 from sqlalchemy.sql.expression import text
 
 from rucio.common import exception
@@ -30,8 +30,14 @@ from rucio.db.sqla.constants import DIDType
 from rucio.db.sqla.session import read_session
 
 if TYPE_CHECKING:
+    from collections.abc import Callable, Iterable
+
     from sqlalchemy.orm import Session
 
+    from rucio.db.sqla.models import ModelBase
+
+    KeyType = TypeVar("KeyType", str, InstrumentedAttribute)
+    FilterTuple = tuple[KeyType, Callable[[object, object], Any], Union[bool, datetime, float, str]]
 
 # lookup table converting keyword suffixes to pythonic operators.
 OPERATORS_CONVERSION_LUT = {
@@ -77,31 +83,36 @@ class FilterEngine:
     """
     An engine to provide advanced filtering functionality to DID listing requests.
     """
-    def __init__(self, filters, model_class=None, strict_coerce=True):
+    def __init__(
+            self,
+            filters: Union[str, dict[str, Any], list[dict[str, Any]]],
+            model_class: Optional[type["ModelBase"]] = None,
+            strict_coerce: bool = True
+    ):
         if isinstance(filters, str):
-            self._filters, _ = parse_did_filter_from_string_fe(filters, omit_name=True)
+            filters, _ = parse_did_filter_from_string_fe(filters, omit_name=True)
         elif isinstance(filters, dict):
-            self._filters = [filters]
+            filters = [filters]
         elif isinstance(filters, list):
-            self._filters = filters
+            filters = filters
         else:
             raise exception.DIDFilterSyntaxError("Input filters are of an unrecognised type.")
 
-        self._make_input_backwards_compatible()
-        self.mandatory_model_attributes = self._translate_filters(model_class=model_class, strict_coerce=strict_coerce)
+        filters = self._make_input_backwards_compatible(filters=filters)
+        self._filters, self.mandatory_model_attributes = self._translate_filters(filters=filters, model_class=model_class, strict_coerce=strict_coerce)
         self._sanity_check_translated_filters()
 
     @property
-    def filters(self):
+    def filters(self) -> list[list["FilterTuple"]]:
         return self._filters
 
-    def _coerce_filter_word_to_model_attribute(self, word, model_class, strict=True):
+    def _coerce_filter_word_to_model_attribute(self, word: Any, model_class: Optional[type["ModelBase"]], strict: bool = True) -> Any:
         """
         Attempts to coerce a filter word to an attribute of a <model_class>.
 
         :param model_class: The word.
         :param model_class: The SQL model class.
-        :params: strict: Enforce that keywords must be coercable to a model attribute.
+        :params: strict: Enforce that keywords must be coercible to a model attribute.
         :returns: The coerced attribute if successful or (if strict is False) the word if not.
         :raises: KeyNotFound
         """
@@ -113,7 +124,7 @@ class FilterEngine:
                     raise exception.KeyNotFound("'{}' keyword could not be coerced to model class attribute. Attribute not found.".format(word))
         return word
 
-    def _make_input_backwards_compatible(self):
+    def _make_input_backwards_compatible(self, filters: list[dict[str, Any]]) -> list[dict[str, Any]]:
         """
         Backwards compatibility for previous versions of filtering.
 
@@ -121,13 +132,14 @@ class FilterEngine:
         - converts "created_after" key to "created_at.gte"
         - converts "created_before" key to "created_at.lte"
         """
-        for or_group in self._filters:
+        for or_group in filters:
             if 'created_after' in or_group:
                 or_group['created_at.gte'] = or_group.pop('created_after')
             elif 'created_before' in or_group:
                 or_group['created_at.lte'] = or_group.pop('created_before')
+        return filters
 
-    def _sanity_check_translated_filters(self):
+    def _sanity_check_translated_filters(self) -> None:
         """
         Perform a few sanity checks on translated filters.
 
@@ -153,7 +165,7 @@ class FilterEngine:
                         raise ValueError("Name operator must be an equality operator.")
                 if key == 'length':     # (3)
                     try:
-                        int(value)
+                        int(value)  # type: ignore
                     except ValueError:
                         raise ValueError('Length has to be an integer value.')
 
@@ -170,7 +182,12 @@ class FilterEngine:
             if len(set(or_group_test_duplicates)) != len(or_group_test_duplicates):     # (6)
                 raise exception.DuplicateCriteriaInDIDFilter()
 
-    def _translate_filters(self, model_class, strict_coerce=True):
+    def _translate_filters(
+            self,
+            filters: "Iterable[dict[str, Any]]",
+            model_class: Optional[type["ModelBase"]],
+            strict_coerce: bool = True
+    ) -> tuple[list[list["FilterTuple"]], list[InstrumentedAttribute[Any]]]:
         """
         Reformats filters from:
 
@@ -199,9 +216,10 @@ class FilterEngine:
 
         Typecasting of values is also attempted.
 
+        :param filters: The filters to translate.
         :param model_class: The SQL model class.
-        :param strict_coerce: Enforce that keywords must be coercable to a model attribute.
-        :returns: The set of mandatory model attributes to be used in the filter query.
+        :param strict_coerce: Enforce that keywords must be coercible to a model attribute.
+        :returns: The list of translated filters, and the set of mandatory model attributes to be used in the filter query.
         :raises: MissingModuleException, DIDFilterSyntaxError
         """
         if model_class:
@@ -212,7 +230,7 @@ class FilterEngine:
 
         mandatory_model_attributes = set()
         filters_translated = []
-        for or_group in self._filters:
+        for or_group in filters:
             and_group_parsed = []
             for key, value in or_group.items():
                 # KEY
@@ -245,10 +263,9 @@ class FilterEngine:
                 and_group_parsed.append(
                     (key_no_suffix, OPERATORS_CONVERSION_LUT.get(oper), value))
             filters_translated.append(and_group_parsed)
-        self._filters = filters_translated
-        return list(mandatory_model_attributes)
+        return filters_translated, list(mandatory_model_attributes)
 
-    def _try_typecast_string(self, value):
+    def _try_typecast_string(self, value: str) -> Union[bool, datetime, float, str]:
         """
         Check if string can be typecasted to bool, datetime or float.
 
@@ -259,11 +276,11 @@ class FilterEngine:
         value = value.replace('false', 'False').replace('FALSE', 'False')
         for format in VALID_DATE_FORMATS:       # try parsing multiple date formats.
             try:
-                value = datetime.strptime(value, format)
+                typecasted_value = datetime.strptime(value, format)
             except ValueError:
                 continue
             else:
-                return value
+                return typecasted_value
         try:
             operators = ('+', '-', '*', '/')
             if not any(operator in value for operator in operators):    # fix for lax ast literal_eval in earlier python versions
@@ -272,21 +289,25 @@ class FilterEngine:
             pass
         return value
 
-    def create_mongo_query(self, additional_filters={}):
+    def create_mongo_query(
+        self,
+        additional_filters: Optional["Iterable[FilterTuple]"] = None
+    ) -> dict[str, Any]:
         """
         Returns a single mongo query describing the filters expression.
 
         :param additional_filters: additional filters to be applied to all clauses.
         :returns: a mongo query string describing the filters expression.
         """
+        additional_filters = additional_filters or []
         # Add additional filters, applied as AND clauses to each OR group.
         for or_group in self._filters:
             for filter in additional_filters:
-                or_group.append(list(filter))
+                or_group.append(list(filter))  # type: ignore
 
-        or_expressions = []
+        or_expressions: list[dict[str, Any]] = []
         for or_group in self._filters:
-            and_expressions = []
+            and_expressions: list[dict[str, dict[str, Any]]] = []
             for and_group in or_group:
                 key, oper, value = and_group
                 if isinstance(value, str) and any([char in value for char in ['*', '%']]):   # wildcards
@@ -327,8 +348,12 @@ class FilterEngine:
 
         return query_str
 
-    def create_postgres_query(self, additional_filters={}, fixed_table_columns=('scope', 'name', 'vo'),
-                              jsonb_column='data'):
+    def create_postgres_query(
+        self,
+        additional_filters: Optional["Iterable[FilterTuple]"] = None,
+        fixed_table_columns: Union[tuple[str, ...], dict[str, str]] = ('scope', 'name', 'vo'),
+        jsonb_column: str = 'data'
+    ) -> str:
         """
         Returns a single postgres query describing the filters expression.
 
@@ -336,14 +361,15 @@ class FilterEngine:
         :param fixed_table_columns: the table columns
         :returns: a postgres query string describing the filters expression.
         """
+        additional_filters = additional_filters or []
         # Add additional filters, applied as AND clauses to each OR group.
         for or_group in self._filters:
             for _filter in additional_filters:
-                or_group.append(list(_filter))
+                or_group.append(list(_filter))  # type: ignore
 
-        or_expressions = []
+        or_expressions: list[str] = []
         for or_group in self._filters:
-            and_expressions = []
+            and_expressions: list[str] = []
             for and_group in or_group:
                 key, oper, value = and_group
                 if key in fixed_table_columns:                                              # is this key filtering on a column or in the jsonb?
@@ -368,7 +394,7 @@ class FilterEngine:
                     # Infer what type key should be cast to from typecasting the value in the expression.
                     try:
                         if isinstance(value, int):                                          # this could be bool or int (as bool subclass of int)
-                            if type(value) == bool:
+                            if isinstance(value, bool):
                                 if is_in_json_column:
                                     expression = "({}->>'{}')::boolean {} {}".format(jsonb_column, key, POSTGRES_OP_MAP[oper], str(value).lower())
                                 else:
@@ -401,7 +427,14 @@ class FilterEngine:
         return ' OR '.join(or_expressions)
 
     @read_session
-    def create_sqla_query(self, *, session: "Session", additional_model_attributes=[], additional_filters={}, json_column=None):
+    def create_sqla_query(
+        self,
+        *,
+        session: "Session",
+        additional_model_attributes: Optional[list[InstrumentedAttribute[Any]]] = None,
+        additional_filters: Optional["Iterable[FilterTuple]"] = None,
+        json_column: Optional[InstrumentedAttribute] = None
+    ) -> Select:
         """
         Returns a database query that fully describes the filters.
 
@@ -412,22 +445,24 @@ class FilterEngine:
         :param additional_model_attributes: Additional model attributes to retrieve.
         :param additional_filters: Additional filters to be applied to all clauses.
         :param json_column: Column to be checked if filter key has not been coerced to a model attribute. Only valid if engine instantiated with strict_coerce=False.
-        :returns: A database query.
+        :returns: A SQLAlchemy Select object.
         :raises: FilterEngineGenericError
         """
+        additional_model_attributes = additional_model_attributes or []
+        additional_filters = additional_filters or []
         all_model_attributes = set(self.mandatory_model_attributes + additional_model_attributes)
 
         # Add additional filters, applied as AND clauses to each OR group.
         for or_group in self._filters:
             for _filter in additional_filters:
-                or_group.append(list(_filter))
+                or_group.append(list(_filter))  # type: ignore
 
-        or_expressions = []
+        or_expressions: list = []
         for or_group in self._filters:
             and_expressions = []
             for and_group in or_group:
                 key, oper, value = and_group
-                if isinstance(key, sqlalchemy.orm.attributes.InstrumentedAttribute):                # -> this key filters on a table column.
+                if isinstance(key, InstrumentedAttribute):                # -> this key filters on a table column.
                     if isinstance(value, str) and any([char in value for char in ['*', '%']]):      # wildcards
                         if value in ('*', '%', '*', '%'):                                           # match wildcard exactly == no filtering on key
                             continue
@@ -473,7 +508,7 @@ class FilterEngine:
                             # Infer what type key should be cast to from typecasting the value in the expression.
                             try:
                                 if isinstance(value, int):                                          # this could be bool or int (as bool subclass of int)
-                                    if type(value) == bool:
+                                    if isinstance(value, bool):
                                         expression = oper(json_column[key].as_boolean(), value)
                                     else:
                                         expression = oper(json_column[key].as_float(), value)       # cast as float, not integer, to avoid potentially losing precision in key
@@ -490,9 +525,14 @@ class FilterEngine:
 
                 and_expressions.append(expression)
             or_expressions.append(and_(*and_expressions))
-        return session.query(*all_model_attributes).filter(or_(*or_expressions))
-
-    def evaluate(self):
+        stmt = select(
+            *all_model_attributes
+        ).where(
+            or_(*or_expressions)
+        )
+        return stmt
+
+    def evaluate(self) -> bool:
         """
         Evaluates an expression and returns a boolean result.
 
@@ -507,7 +547,7 @@ class FilterEngine:
             or_group_evaluations.append(all(and_group_evaluations))
         return any(or_group_evaluations)
 
-    def print_filters(self):
+    def print_filters(self) -> str:
         """
         A (more) human readable format of <filters>.
         """
@@ -517,16 +557,16 @@ class FilterEngine:
         for or_group in self._filters:
             for and_group in or_group:
                 key, oper, value = and_group
-                if isinstance(key, sqlalchemy.orm.attributes.InstrumentedAttribute):
+                if isinstance(key, InstrumentedAttribute):
                     key = and_group[0].key
                 if operators_conversion_LUT_inv[oper] == "":
                     oper = "eq"
                 else:
                     oper = operators_conversion_LUT_inv[oper]
-                if isinstance(value, sqlalchemy.orm.attributes.InstrumentedAttribute):
-                    value = and_group[2].key
+                if isinstance(value, InstrumentedAttribute):
+                    value = and_group[2].key  # type: ignore
                 elif isinstance(value, DIDType):
-                    value = and_group[2].name
+                    value = and_group[2].name  # type: ignore
                 filters = "{}{} {} {}".format(filters, key, oper, value)
                 if and_group != or_group[-1]:
                     filters += ' AND '