rucio 35.7.0__py3-none-any.whl → 37.0.0rc2__py3-none-any.whl

rucio/core/account_limit.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Optional, Union
 
 from sqlalchemy.exc import NoResultFound
 from sqlalchemy.sql import func, literal, select
@@ -27,9 +27,11 @@ from rucio.db.sqla.session import read_session, transactional_session
 if TYPE_CHECKING:
     from sqlalchemy.orm import Session
 
+    from rucio.common.types import InternalAccount, RSEAccountUsageDict, RSEGlobalAccountUsageDict, RSELocalAccountUsageDict, RSEResolvedGlobalAccountLimitDict
+
 
 @read_session
-def get_rse_account_usage(rse_id, *, session: "Session"):
+def get_rse_account_usage(rse_id: str, *, session: "Session") -> list["RSEAccountUsageDict"]:
     """
     Returns the account limit and usage for all accounts on a RSE.
 
@@ -89,7 +91,7 @@ def get_rse_account_usage(rse_id, *, session: "Session"):
 
 
 @read_session
-def get_global_account_limits(account=None, *, session: "Session"):
+def get_global_account_limits(account: Optional["InternalAccount"] = None, *, session: "Session") -> dict[str, "RSEResolvedGlobalAccountLimitDict"]:
     """
     Returns the global account limits for the account.
 
@@ -128,7 +130,7 @@ def get_global_account_limits(account=None, *, session: "Session"):
 
 
 @read_session
-def get_global_account_limit(account, rse_expression, *, session: "Session"):
+def get_global_account_limit(account: "InternalAccount", rse_expression: str, *, session: "Session") -> Union[int, float, None]:
     """
     Returns the global account limit for the account on the rse expression.
 
@@ -154,7 +156,7 @@ def get_global_account_limit(account, rse_expression, *, session: "Session"):
 
 
 @read_session
-def get_local_account_limit(account, rse_id, *, session: "Session"):
+def get_local_account_limit(account: "InternalAccount", rse_id: str, *, session: "Session") -> Union[int, float, None]:
     """
     Returns the account limit for the account on the rse.
 
@@ -180,7 +182,7 @@ def get_local_account_limit(account, rse_id, *, session: "Session"):
 
 
 @read_session
-def get_local_account_limits(account, rse_ids=None, *, session: "Session"):
+def get_local_account_limits(account: "InternalAccount", rse_ids: Optional[list[str]] = None, *, session: "Session") -> dict[str, int]:
     """
     Returns the account limits for the account on the list of rses.
 
@@ -225,7 +227,7 @@ def get_local_account_limits(account, rse_ids=None, *, session: "Session"):
 
 
 @transactional_session
-def set_local_account_limit(account, rse_id, bytes_, *, session: "Session"):
+def set_local_account_limit(account: "InternalAccount", rse_id: str, bytes_: int, *, session: "Session") -> None:
     """
     Returns the limits for the account on the rse.
 
@@ -248,7 +250,7 @@ def set_local_account_limit(account, rse_id, bytes_, *, session: "Session"):
 
 
 @transactional_session
-def set_global_account_limit(account, rse_expression, bytes_, *, session: "Session"):
+def set_global_account_limit(account: "InternalAccount", rse_expression: str, bytes_: int, *, session: "Session") -> None:
     """
     Sets the global limit for the account on a RSE expression.
 
@@ -271,7 +273,7 @@ def set_global_account_limit(account, rse_expression, bytes_, *, session: "Sessi
 
 
 @transactional_session
-def delete_local_account_limit(account, rse_id, *, session: "Session"):
+def delete_local_account_limit(account: "InternalAccount", rse_id: str, *, session: "Session") -> bool:
     """
     Deletes a local account limit.
 
@@ -295,7 +297,7 @@ def delete_local_account_limit(account, rse_id, *, session: "Session"):
 
 
 @transactional_session
-def delete_global_account_limit(account, rse_expression, *, session: "Session"):
+def delete_global_account_limit(account: "InternalAccount", rse_expression: str, *, session: "Session") -> bool:
     """
     Deletes a global account limit.
 
@@ -319,7 +321,7 @@ def delete_global_account_limit(account, rse_expression, *, session: "Session"):
 
 
 @transactional_session
-def get_local_account_usage(account, rse_id=None, *, session: "Session"):
+def get_local_account_usage(account: "InternalAccount", rse_id: Optional[str] = None, *, session: "Session") -> list["RSELocalAccountUsageDict"]:
     """
     Read the account usage and connect it with (if available) the account limits of the account.
 
@@ -370,7 +372,7 @@ def get_local_account_usage(account, rse_id=None, *, session: "Session"):
 
 
 @transactional_session
-def get_global_account_usage(account, rse_expression=None, *, session: "Session"):
+def get_global_account_usage(account: "InternalAccount", rse_expression: Optional[str] = None, *, session: "Session") -> list["RSEGlobalAccountUsageDict"]:
     """
     Read the account usage and connect it with the global account limits of the account.
 

rucio/core/authentication.py

@@ -26,7 +26,7 @@ from dogpile.cache import make_region
 from dogpile.cache.api import NO_VALUE, NoValue
 from sqlalchemy import delete, null, or_, select
 
-from rucio.common.cache import make_region_memcached
+from rucio.common.cache import MemcacheRegion
 from rucio.common.config import config_get_bool
 from rucio.common.exception import CannotAuthenticate, RucioException
 from rucio.common.utils import chunks, date_to_str, generate_uuid
@@ -83,7 +83,7 @@ def token_key_generator(namespace, fni, **kwargs):
 
 
 if config_get_bool('cache', 'use_external_cache_for_auth_tokens', default=False):
-    TOKENREGION = make_region_memcached(expiration_time=900, function_key_generator=token_key_generator)
+    TOKENREGION = MemcacheRegion(expiration_time=900, function_key_generator=token_key_generator)
 else:
     TOKENREGION = make_region(function_key_generator=token_key_generator).configure('dogpile.cache.memory', expiration_time=900)
 

rucio/core/config.py

@@ -12,13 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from collections.abc import Callable
 from typing import TYPE_CHECKING, Any, Optional, TypeVar
 
 from dogpile.cache.api import NoValue
 from sqlalchemy import and_, delete, func, select, update
 
-from rucio.common.cache import make_region_memcached
+from rucio.common.cache import CacheKey, MemcacheRegion
 from rucio.common.exception import ConfigNotFound
 from rucio.db.sqla import models
 from rucio.db.sqla.session import read_session, transactional_session
@@ -26,34 +25,16 @@ from rucio.db.sqla.session import read_session, transactional_session
 T = TypeVar('T')
 
 if TYPE_CHECKING:
+    from collections.abc import Callable
+
     from sqlalchemy.orm import Session
 
 
-REGION = make_region_memcached(expiration_time=900)
+REGION = MemcacheRegion(expiration_time=900)
 
 SECTIONS_CACHE_KEY = 'sections'
 
 
-def _has_section_cache_key(section: str) -> str:
-    return 'has_section_%s' % section
-
-
-def _options_cache_key(section: str) -> str:
-    return 'options_%s' % section
-
-
-def _has_option_cache_key(section: str, option: str) -> str:
-    return 'has_option_%s_%s' % (section, option)
-
-
-def _items_cache_key(section: str) -> str:
-    return 'items_%s' % section
-
-
-def _value_cache_key(section: str, option: str) -> str:
-    return 'get_%s_%s' % (section, option)
-
-
 @read_session
 def sections(
         *,
@@ -144,7 +125,7 @@ def options(
     :param session: The database session in use.
     :returns: ['option', ...]
     """
-    options_key = _options_cache_key(section)
+    options_key = CacheKey.options(section)
     options = NoValue()
     if use_cache:
         options = read_from_cache(options_key, expiration_time)
@@ -178,7 +159,7 @@ def has_option(
     :param session: The database session in use.
     :returns: True/False
     """
-    has_option_key = _has_option_cache_key(section, option)
+    has_option_key = CacheKey.has_option(section, option)
     has_option = NoValue()
     if use_cache:
         has_option = read_from_cache(has_option_key, expiration_time)
@@ -202,7 +183,7 @@ def get(
         default: Optional[T] = None,
         use_cache: bool = True,
         expiration_time: int = 900,
-        convert_type_fnc: Callable[[str], T],
+        convert_type_fnc: 'Callable[[str], T]',
         session: "Session"
 ) -> T:
     """
@@ -220,7 +201,7 @@ def get(
     :param session: The database session in use.
     :returns: The auto-coerced value.
     """
-    value_key = _value_cache_key(section, option)
+    value_key = CacheKey.value(section, option)
     value = NoValue()
     if use_cache:
         value = read_from_cache(value_key, expiration_time)
@@ -251,7 +232,7 @@ def items(
         use_cache: bool = True,
         expiration_time: int = 900,
         *,
-        convert_type_fnc: Callable[[str], T],
+        convert_type_fnc: 'Callable[[str], T]',
         session: "Session"
 ) -> list[tuple[str, T]]:
     """
@@ -264,7 +245,7 @@ def items(
     :param session: The database session in use.
     :returns: [('option', auto-coerced value), ...]
     """
-    items_key = _items_cache_key(section)
+    items_key = CacheKey.items(section)
     items = NoValue()
     if use_cache:
         items = read_from_cache(items_key, expiration_time)
@@ -303,12 +284,12 @@ def set(
         new_option = models.Config(section=section, opt=option, value=value)
         new_option.save(session=session)
 
-        delete_from_cache(key=_value_cache_key(section, option))
-        delete_from_cache(key=_has_option_cache_key(section, option))
-        delete_from_cache(key=_items_cache_key(section))
+        delete_from_cache(key=CacheKey.value(section, option))
+        delete_from_cache(key=CacheKey.has_option(section, option))
+        delete_from_cache(key=CacheKey.items(section))
         if not section_existed:
             delete_from_cache(key=SECTIONS_CACHE_KEY)
-            delete_from_cache(key=_has_section_cache_key(section))
+            delete_from_cache(key=CacheKey.has_section(section))
     else:
         stmt = select(
             models.Config.value
@@ -331,8 +312,8 @@ def set(
                 models.Config.value: str(value)
             })
             session.execute(stmt)
-            delete_from_cache(key=_value_cache_key(section, option))
-            delete_from_cache(key=_items_cache_key(section))
+            delete_from_cache(key=CacheKey.value(section, option))
+            delete_from_cache(key=CacheKey.items(section))
 
 
 @transactional_session
@@ -358,8 +339,8 @@ def remove_section(section: str, *, session: "Session") -> bool:
                                               opt=old[1],
                                               value=old[2])
             old_option.save(session=session)
-            delete_from_cache(key=_has_option_cache_key(old[0], old[1]))
-            delete_from_cache(key=_value_cache_key(old[0], old[1]))
+            delete_from_cache(key=CacheKey.has_option(old[0], old[1]))
+            delete_from_cache(key=CacheKey.value(old[0], old[1]))
 
         stmt = delete(
             models.Config
@@ -368,7 +349,7 @@ def remove_section(section: str, *, session: "Session") -> bool:
         )
         session.execute(stmt)
         delete_from_cache(key=SECTIONS_CACHE_KEY)
-        delete_from_cache(key=_items_cache_key(section))
+        delete_from_cache(key=CacheKey.items(section))
         return True
 
 
@@ -416,10 +397,10 @@ def remove_option(section: str, option: str, *, session: "Session") -> bool:
         if not session.execute(stmt).scalar_one_or_none():
             # we deleted the last config entry in the section. Invalidate the section cache
             delete_from_cache(key=SECTIONS_CACHE_KEY)
-            delete_from_cache(key=_has_section_cache_key(section))
-        delete_from_cache(key=_items_cache_key(section))
-        delete_from_cache(key=_has_option_cache_key(section, option))
-        delete_from_cache(key=_value_cache_key(section, option))
+            delete_from_cache(key=CacheKey.has_section(section))
+        delete_from_cache(key=CacheKey.items(section))
+        delete_from_cache(key=CacheKey.has_option(section, option))
+        delete_from_cache(key=CacheKey.value(section, option))
         return True
 
 

rucio/core/credential.py

@@ -17,7 +17,7 @@ import datetime
 import hmac
 import time
 from hashlib import sha1
-from typing import Literal
+from typing import Optional
 from urllib.parse import urlencode, urlparse
 
 import boto3
@@ -25,25 +25,25 @@ from botocore.client import Config
 from dogpile.cache.api import NO_VALUE
 from google.oauth2.service_account import Credentials
 
-from rucio.common.cache import make_region_memcached
+from rucio.common.cache import MemcacheRegion
 from rucio.common.config import config_get, get_rse_credentials
-from rucio.common.constants import RseAttr
+from rucio.common.constants import RSE_BASE_SUPPORTED_PROTOCOL_OPERATIONS, RSE_BASE_SUPPORTED_PROTOCOL_OPERATIONS_LITERAL, SUPPORTED_SIGN_URL_SERVICES, SUPPORTED_SIGN_URL_SERVICES_LITERAL, RseAttr
 from rucio.common.exception import UnsupportedOperation
 from rucio.core.monitor import MetricManager
 from rucio.core.rse import get_rse_attribute
 
 CREDS_GCS = None
 
-REGION = make_region_memcached(expiration_time=900)
+REGION = MemcacheRegion(expiration_time=900)
 METRICS = MetricManager(module=__name__)
 
 
 def get_signed_url(
         rse_id: str,
-        service: Literal['gsc', 's3', 'swift'],
-        operation: Literal['read', 'write', 'delete'],
+        service: SUPPORTED_SIGN_URL_SERVICES_LITERAL,
+        operation: RSE_BASE_SUPPORTED_PROTOCOL_OPERATIONS_LITERAL,
         url: str,
-        lifetime: int = 600
+        lifetime: Optional[int] = 600
 ) -> str:
     """
     Get a signed URL for a particular service and operation.
@@ -60,10 +60,10 @@ def get_signed_url(
 
     global CREDS_GCS
 
-    if service not in ['gcs', 's3', 'swift']:
+    if service not in SUPPORTED_SIGN_URL_SERVICES:
         raise UnsupportedOperation('Service must be "gcs", "s3" or "swift"')
 
-    if operation not in ['read', 'write', 'delete']:
+    if operation not in RSE_BASE_SUPPORTED_PROTOCOL_OPERATIONS:
         raise UnsupportedOperation('Operation must be "read", "write", or "delete"')
 
     if url is None or url == '':
@@ -84,10 +84,6 @@ def get_signed_url(
         components = urlparse(url)
         host = components.netloc
 
-        # select the correct operation
-        operations = {'read': 'GET', 'write': 'PUT', 'delete': 'DELETE'}
-        operation = operations[operation]
-
         # special case to test signature, force epoch time
         if lifetime is None:
             lifetime = 0
@@ -100,8 +96,11 @@ def get_signed_url(
         # sign the path only
         path = components.path
 
+        # Map operations
+        operations = {'read': 'GET', 'write': 'PUT', 'delete': 'DELETE'}
+
         # assemble message to sign
-        to_sign = "%s\n\n\n%s\n%s" % (operation, lifetime, path)
+        to_sign = "%s\n\n\n%s\n%s" % (operations[operation], lifetime, path)
 
         # create URL-capable signature
         # first character is always a '=', remove it
@@ -213,7 +212,7 @@ def get_signed_url(
         else:
             swiftop = 'DELETE'
 
-        expires = int(time.time() + lifetime)
+        expires = int(time.time() + lifetime)  # type: ignore (lifetime could be None)
 
         # create signed URL
         with METRICS.timer('signswift'):

rucio/core/did.py

@@ -2011,8 +2011,10 @@ def get_metadata(
 
     :param scope: The scope name.
     :param name: The data identifier name.
+    :param plugin: The metadata plugin to use or 'ALL' for all.
     :param session: The database session in use.
 
+
     :returns: List of HARDCODED metadata for did.
     """
     return did_meta_plugins.get_metadata(scope, name, plugin=plugin, session=session)
@@ -2058,6 +2060,7 @@ def list_parent_dids_bulk(
 def get_metadata_bulk(
     dids: list["Mapping[Any, Any]"],
     inherit: bool = False,
+    plugin: str = 'JSON',
     *,
     session: "Session"
 ) -> "Iterator[dict[str, Any]]":
@@ -2065,6 +2068,7 @@ def get_metadata_bulk(
     Get metadata for a list of dids
     :param dids:               A list of dids.
     :param inherit:            A boolean. If set to true, the metadata of the parent are concatenated.
+    :param plugin:             A string. The metadata plugin to use or 'ALL' for all.
     :param session:            The database session in use.
     """
     if inherit:
@@ -2096,7 +2100,7 @@ def get_metadata_bulk(
         meta_dict = {}
         for did in unique_dids:
             try:
-                meta = get_metadata(did['scope'], did['name'], plugin='JSON', session=session)
+                meta = get_metadata(did['scope'], did['name'], plugin=plugin, session=session)
             except exception.DataIdentifierNotFound:
                 meta = {}
             meta_dict[(did['scope'], did['name'])] = meta