rucio 37.7.0__py3-none-any.whl → 38.0.0__py3-none-any.whl

rucio/gateway/opendata.py

@@ -0,0 +1,190 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+from typing import TYPE_CHECKING, Any, Optional
+
+from rucio.common.constants import DEFAULT_VO
+from rucio.common.types import InternalScope
+from rucio.common.utils import gateway_update_return_dict
+from rucio.core import opendata
+from rucio.core.opendata import opendata_state_str_to_enum, validate_opendata_did_state
+from rucio.db.sqla.constants import DatabaseOperationType
+from rucio.db.sqla.session import db_session
+
+if TYPE_CHECKING:
+    from rucio.common.constants import OPENDATA_DID_STATE_LITERAL
+
+
+def list_opendata_dids(
+        *,
+        limit: Optional[int] = None,
+        offset: Optional[int] = None,
+        state: Optional["OPENDATA_DID_STATE_LITERAL"] = None,
+) -> dict[str, list[dict[str, Any]]]:
+    """
+    List Opendata DIDs from the Opendata catalog.
+
+    Parameters:
+        limit: Maximum number of DIDs to return.
+        offset: Number of DIDs to skip before starting to collect the result set.
+        state: Filter DIDs by their state.
+
+    Returns:
+        A dictionary with a list of DIDs matching the criteria.
+    """
+
+    state_enum = None
+    if state is not None:
+        state = validate_opendata_did_state(state)
+        state_enum = opendata_state_str_to_enum(state)
+    with db_session(DatabaseOperationType.READ) as session:
+        result = opendata.list_opendata_dids(limit=limit, offset=offset, state=state_enum, session=session)
+    return result
+
+
+def get_opendata_did(
+        *,
+        scope: str,
+        name: str,
+        state: Optional["OPENDATA_DID_STATE_LITERAL"] = None,
+        include_files: bool = True,
+        include_metadata: bool = False,
+        include_doi: bool = True,
+        vo: str = DEFAULT_VO,
+) -> dict[str, Any]:
+    """
+    Retrieve a specific Opendata DID from the Opendata catalog.
+
+    Parameters:
+        scope: The scope of the DID.
+        name: The name of the DID.
+        state: Optional state to filter the DID.
+        include_files: Whether to include files in the result.
+        include_metadata: Whether to include metadata in the result.
+        include_doi: Whether to include DOI information in the result.
+        vo: The virtual organization.
+
+    Returns:
+        A dictionary containing the details of the requested DID.
+    """
+
+    internal_scope = InternalScope(scope, vo=vo)
+    state_enum = None
+    if state is not None:
+        state = validate_opendata_did_state(state)
+        state_enum = opendata_state_str_to_enum(state)
+
+    with db_session(DatabaseOperationType.READ) as session:
+        result = opendata.get_opendata_did(scope=internal_scope,
+                                           name=name,
+                                           state=state_enum,
+                                           include_files=include_files,
+                                           include_metadata=include_metadata,
+                                           include_doi=include_doi,
+                                           session=session)
+        return gateway_update_return_dict(result, session=session)
+
+
+def add_opendata_did(
+        *,
+        scope: str,
+        name: str,
+        vo: str = DEFAULT_VO,
+) -> None:
+    """
+    Add a new Opendata DID to the Opendata catalog.
+
+    Parameters:
+        scope: The scope of the DID.
+        name: The name of the DID.
+        vo: The virtual organization.
+
+    Returns:
+        None
+    """
+
+    internal_scope = InternalScope(scope, vo=vo)
+    with db_session(DatabaseOperationType.WRITE) as session:
+        return opendata.add_opendata_did(scope=internal_scope, name=name, session=session)
+
+
+def delete_opendata_did(
+        *,
+        scope: str,
+        name: str,
+        vo: str = DEFAULT_VO,
+) -> None:
+    """
+    Delete an Opendata DID from the Opendata catalog.
+
+    Parameters:
+        scope: The scope of the DID.
+        name: The name of the DID.
+        vo: The virtual organization.
+
+    Returns:
+        None
+    """
+
+    internal_scope = InternalScope(scope, vo=vo)
+    with db_session(DatabaseOperationType.WRITE) as session:
+        return opendata.delete_opendata_did(scope=internal_scope, name=name, session=session)
+
+
+def update_opendata_did(
+        *,
+        scope: str,
+        name: str,
+        state: Optional["OPENDATA_DID_STATE_LITERAL"] = None,
+        meta: Optional[dict] = None,
+        doi: Optional[str] = None,
+        vo: str = DEFAULT_VO,
+) -> None:
+    """
+    Update an existing Opendata DID in the Opendata catalog.
+
+    Parameters:
+        scope: The scope of the DID.
+        name: The name of the DID.
+        state: Optional new state for the DID.
+        meta: Optional metadata dictionary or JSON string.
+        doi: Optional DOI string.
+        vo: The virtual organization.
+
+    Returns:
+        None
+
+    Raises:
+        ValueError: If meta is a string and cannot be parsed as valid JSON.
+    """
+
+    internal_scope = InternalScope(scope, vo=vo)
+    state_enum = None
+    if state is not None:
+        state = validate_opendata_did_state(state)
+        state_enum = opendata_state_str_to_enum(state)
+    if isinstance(meta, str):
+        try:
+            meta = json.loads(meta)
+        except ValueError as error:
+            raise ValueError(f"Invalid JSON: {error}")
+
+    with db_session(DatabaseOperationType.WRITE) as session:
+        return opendata.update_opendata_did(scope=internal_scope,
+                                            name=name,
+                                            state=state_enum,
+                                            meta=meta,
+                                            doi=doi,
+                                            session=session)

rucio/gateway/subscription.py

@@ -21,7 +21,7 @@ from rucio.common.exception import AccessDenied, InvalidObject
 from rucio.common.schema import validate_schema
 from rucio.common.types import InternalAccount, InternalScope
 from rucio.core import subscription
-from rucio.db.sqla.constants import DatabaseOperationType
+from rucio.db.sqla.constants import DatabaseOperationType, SubscriptionState
 from rucio.db.sqla.session import db_session
 from rucio.gateway.permission import has_permission
 
@@ -110,7 +110,7 @@ def update_subscription(
 
     :param name: Name of the subscription
     :param account: Account identifier
-    :param metadata: Dictionary of metadata to update. Supported keys : filter, replication_rules, comments, lifetime, retroactive, dry_run, priority, last_processed
+    :param metadata: Dictionary of metadata to update. Supported keys : filter, replication_rules, comments, lifetime, retroactive, dry_run, priority, last_processed, state
     :param issuer: The account issuing this operation.
     :param vo: The VO to act on.
     :raises: SubscriptionNotFound if subscription is not found
@@ -132,6 +132,9 @@ def update_subscription(
                 else:
                     for rule in metadata['replication_rules']:
                         validate_schema(name='activity', obj=rule.get('activity', 'default'), vo=vo)
+            if 'state' in metadata and metadata['state'] is not None:
+                metadata['state'] = SubscriptionState(metadata['state'])
+
         except ValueError as error:
             raise TypeError(error)
 
@@ -148,7 +151,6 @@ def update_subscription(
                         filter_[_key] = [_type(val, vo=vo).internal for val in filter_[_key]]
                     else:
                         filter_[_key] = _type(filter_[_key], vo=vo).internal
-
         return subscription.update_subscription(name=name, account=internal_account, metadata=metadata, session=session)
 
 

rucio/rse/protocols/protocol.py

@@ -69,11 +69,15 @@ class RSEProtocol(ABC):
         self.rse = rse_settings
         self.logger = logger
         if self.rse['deterministic']:
-            self.translator = RSEDeterministicTranslation(self.rse['rse'], rse_settings, self.attributes)
-            if getattr(rsemanager, 'CLIENT_MODE', None) and \
-                    not RSEDeterministicTranslation.supports(self.rse.get('lfn2pfn_algorithm')):
-                # Remote server has an algorithm we don't understand; always make the server do the lookup.
-                setattr(self, 'lfns2pfns', self.__lfns2pfns_client)
+            if getattr(rsemanager, 'SERVER_MODE', None):
+                vo = get_rse_vo(self.rse['id'])
+            if getattr(rsemanager, 'CLIENT_MODE', None):
+                # assume client has only one VO policy package configured
+                vo = ''
+                if not RSEDeterministicTranslation.supports(self.rse.get('lfn2pfn_algorithm')):
+                    # Remote server has an algorithm we don't understand; always make the server do the lookup.
+                    setattr(self, 'lfns2pfns', self.__lfns2pfns_client)
+            self.translator = RSEDeterministicTranslation(self.rse['rse'], rse_settings, self.attributes, vo)
         else:
             if getattr(rsemanager, 'CLIENT_MODE', None):
                 setattr(self, 'lfns2pfns', self.__lfns2pfns_client)

rucio/rse/translation.py

@@ -50,7 +50,7 @@ class RSEDeterministicScopeTranslation(PolicyPackageAlgorithms):
                 algorithm_name = "def"
             logger.debug("PFN2LFN: Falling back to %s algorithm.", 'default' if algorithm_name == 'def' else algorithm_name)
 
-        self.parser = self.get_parser(algorithm_name)
+        self.parser = self.get_parser(algorithm_name, vo)
 
     @classmethod
     def _module_init_(cls) -> None:
@@ -60,8 +60,14 @@ class RSEDeterministicScopeTranslation(PolicyPackageAlgorithms):
         cls.register(cls._default, "def")
 
     @classmethod
-    def get_parser(cls, algorithm_name: str) -> 'Callable[..., Any]':
-        return super()._get_one_algorithm(cls._algorithm_type, algorithm_name)
+    def get_parser(cls, algorithm_name: str, vo: str) -> 'Callable[..., Any]':
+        result = None
+        if algorithm_name == vo:
+            # default algorithm for VO
+            result = super()._get_default_algorithm(RSEDeterministicScopeTranslation._algorithm_type, vo)
+        if result is None:
+            result = super()._get_one_algorithm(cls._algorithm_type, algorithm_name)
+        return result
 
     @classmethod
     def register(
@@ -111,7 +117,8 @@ class RSEDeterministicTranslation(PolicyPackageAlgorithms):
             self,
             rse: Optional[str] = None,
             rse_attributes: Optional["RSESettingsDict"] = None,
-            protocol_attributes: Optional[dict[str, Any]] = None
+            protocol_attributes: Optional[dict[str, Any]] = None,
+            vo: str = DEFAULT_VO
     ):
         """
         Initialize a translator object from the RSE, its attributes, and the protocol-specific
@@ -125,6 +132,7 @@ class RSEDeterministicTranslation(PolicyPackageAlgorithms):
         self.rse = rse
         self.rse_attributes = rse_attributes if rse_attributes else {}
         self.protocol_attributes = protocol_attributes if protocol_attributes else {}
+        self.vo = vo
 
     @classmethod
     def supports(
@@ -251,9 +259,12 @@ class RSEDeterministicTranslation(PolicyPackageAlgorithms):
             :returns: RSE specific URI of the physical file
         """
         algorithm = self.rse_attributes.get(RseAttr.LFN2PFN_ALGORITHM, 'default')
-        if algorithm == 'default':
+        algorithm_callable = None
+        if algorithm == 'default' or algorithm == RSEDeterministicTranslation._DEFAULT_LFN2PFN:
             algorithm = RSEDeterministicTranslation._DEFAULT_LFN2PFN
-        algorithm_callable = super()._get_one_algorithm(RSEDeterministicTranslation._algorithm_type, algorithm)
+            algorithm_callable = super()._get_default_algorithm(RSEDeterministicTranslation._algorithm_type, self.vo)
+        if algorithm_callable is None:
+            algorithm_callable = super()._get_one_algorithm(RSEDeterministicTranslation._algorithm_type, algorithm)
         return algorithm_callable(scope, name, self.rse, self.rse_attributes, self.protocol_attributes)
 
 

rucio/tests/common.py

@@ -50,23 +50,75 @@ skip_outside_gh_actions = pytest.mark.skipif(os.getenv("GITHUB_ACTIONS") != "tru
                                              reason="Skipping tests outside GitHub Actions")
 
 
-def is_influxdb_available() -> bool:
-    """Return True if influxdb is available, else return False."""
+def is_influxdb_available(
+        url: str = "http://influxdb:8086",
+        timeout: float = 2.0
+) -> bool:
+    """
+    Return True when InfluxDB is up and ready for queries, otherwise False.
+
+    Strategy:
+    1. Try /health           → 200 + JSON["status"] == "pass"
+    2. Fallback to /ping     → 204
+    """
+    print(f"Checking InfluxDB availability at {url}")
+    try:
+        r = requests.get(f"{url}/health", timeout=timeout)
+        print(f"InfluxDB /health responded with {r.status_code} and body: {r.text}", r.status_code, r.text)
+        if r.status_code == 200 and r.json().get("status") == "pass":
+            return True
+        print(f"InfluxDB is not running healthy at {url}.")
+        return False
+    except requests.RequestException as e:
+        # /health failed or is not available (pre‑1.8)
+        print(f"Failed to query InfluxDB /health at {url}: {e}")
+
     try:
-        response = requests.get('http://localhost:8086/ping')
-        return response.status_code == 204
-    except requests.exceptions.ConnectionError:
-        print('InfluxDB is not running at localhost:8086')
+        print(f"Falling back to /ping for InfluxDB at {url}")
+        r = requests.get(f"{url}/ping", timeout=timeout)
+        print(f"InfluxDB /ping responded with {r.status_code}")
+        return r.status_code == 204
+    except requests.RequestException as e:
+        print(f"InfluxDB is not reachable at {url}: {e}")
         return False
 
 
-def is_elasticsearch_available() -> bool:
-    """Return True if elasticsearch is available, else return False."""
+def is_elasticsearch_available(
+        url: str = "http://elasticsearch:9200",
+        timeout: float = 2.0,
+        min_status: str = 'green',
+) -> bool:
+    """
+    Return True when the Elasticsearch node is reachable **and**
+    cluster health is at least `min_status` ('red'<'yellow'<'green').
+
+    1. GET /_cluster/health  → 200 + JSON["status"] meets threshold
+    2. Fallback: HEAD /      → 200 (port open but health unknown)
+    """
+    _status_level = {"red": 1, "yellow": 2, "green": 3}
+
+    print(f"Checking Elasticsearch availability at {url}")
+    try:
+        r = requests.get(f"{url}/_cluster/health", timeout=timeout)
+        print(f"Elasticsearch /_cluster/health responded with {r.status_code} and body: {r.text}")
+        if r.status_code == 200:
+            status = r.json().get("status")
+            if status and _status_level[status] >= _status_level[min_status]:
+                return True
+            print(f"Elasticsearch health is {status!r}, below threshold {min_status!r}.")
+            return False
+    except requests.RequestException as e:
+        # Either not reachable or /_cluster/health not yet available
+        print(f"Failed to query Elasticsearch /_cluster/health at {url}: {e}")
+
+    # Very old nodes or boot‑strapping clusters: fall back to a simple HEAD /
     try:
-        response = requests.get('http://localhost:9200/')
-        return response.status_code == 200
-    except requests.exceptions.ConnectionError:
-        print('Elasticsearch is not running at localhost:9200')
+        print(f"Falling back to HEAD request for Elasticsearch at {url}")
+        r = requests.head(url, timeout=timeout)
+        print(f"Elasticsearch HEAD / responded with {r.status_code}")
+        return r.status_code == 200
+    except requests.RequestException as e:
+        print(f"Elasticsearch is not reachable at {url}: {e}")
         return False
 
 

rucio/transfertool/fts3.py

@@ -1046,6 +1046,7 @@ class FTS3Transfertool(Transfertool):
                 t_file['scitag'] = self.scitags_exp_id << 6 | activity_id
 
         if t_file['metadata']['dst_type'] == 'TAPE':
+            t_file['metadata']['vo'] = rws.scope.vo
             for plugin in self.tape_metadata_plugins:
                 t_file = deep_merge_dict(source=plugin.hints(t_file['metadata']), destination=t_file)
 

rucio/transfertool/fts3_plugins.py

@@ -17,6 +17,7 @@ import sys
 from typing import TYPE_CHECKING, Any, Optional, TypeVar
 
 from rucio.common.config import config_get_int
+from rucio.common.constants import DEFAULT_VO
 from rucio.common.exception import InvalidRequest
 from rucio.common.plugins import PolicyPackageAlgorithms
 
@@ -85,7 +86,11 @@ class FTS3TapeMetadataPlugin(PolicyPackageAlgorithms):
         """
         return {"collocation_hints": collocation_func(**hints)}
 
-    def _default(self, *hints: dict) -> dict:
+    def _default(self, hint_dict: dict[str, Any]) -> dict:
+        vo = hint_dict['vo'] if 'vo' in hint_dict else DEFAULT_VO
+        default_algorithm = self._get_default_algorithm(self.ALGORITHM_NAME, vo=vo)
+        if default_algorithm is not None:
+            return default_algorithm(hint_dict)
         return {}
 
     def _verify_in_format(self, hint_dict: dict[str, Any]) -> None:

rucio/vcsversion.py

@@ -4,8 +4,8 @@ This file is automatically generated; Do not edit it. :)
 '''
 VERSION_INFO = {
     'final': True,
-    'version': '37.7.0',
-    'branch_nick': 'release-37',
-    'revision_id': '61ed028cbd532ede88189a81bee971db566fcdec',
-    'revno': 13826
+    'version': '38.0.0',
+    'branch_nick': 'master',
+    'revision_id': 'f355202c14bffc0dff7c4f83c86c90fcce914472',
+    'revno': 13928
 }

rucio/web/rest/flaskapi/v1/auth.py

@@ -19,6 +19,7 @@ from typing import TYPE_CHECKING
 from urllib.parse import urlparse
 
 from flask import Blueprint, Flask, Response, redirect, render_template, request
+from jinja2.exceptions import TemplateNotFound
 from werkzeug.datastructures import Headers
 
 from rucio.common.config import config_get
@@ -579,9 +580,17 @@ class CodeOIDC(ErrorHandlingMethodView):
             return render_template('auth_crash.html', crashtype='no_result'), 401, headers
 
         if 'fetchcode' in result:
-            return render_template('auth_granted.html', authcode=result['fetchcode']), 200, headers
+            try:
+                return render_template('auth_granted.html', authcode=result['fetchcode']), 200, headers
+            except TemplateNotFound:
+                headers.set('Content-Type', 'text/plain')
+                return 'auth_granted.html missing', 500, headers
         elif 'polling' in result and result['polling'] is True:
-            return render_template('auth_granted.html', authcode='allok'), 200, headers
+            try:
+                return render_template('auth_granted.html', authcode='allok'), 200, headers
+            except TemplateNotFound:
+                headers.set('Content-Type', 'text/plain')
+                return 'auth_granted.html missing', 500, headers
         else:
             headers.extend(error_headers('InvalidRequest', 'Cannot recognize and process your request'))
             return render_template('auth_crash.html', crashtype='bad_request'), 400, headers

rucio/web/rest/flaskapi/v1/common.py

@@ -20,11 +20,12 @@ import re
 from configparser import NoOptionError, NoSectionError
 from functools import wraps
 from time import time
-from typing import TYPE_CHECKING, Any, Literal, Optional, TypeVar, Union
+from typing import TYPE_CHECKING, Any, Literal, Optional, TypeVar, Union, cast
 from urllib.parse import unquote_plus
 
 import flask
 from flask.views import MethodView
+from typing_extensions import ParamSpec
 from werkzeug.datastructures import Headers
 from werkzeug.exceptions import HTTPException
 from werkzeug.wrappers import Request, Response
@@ -183,25 +184,44 @@ def response_headers(response: ResponseTypeVar) -> ResponseTypeVar:
     return response
 
 
-def check_accept_header_wrapper_flask(supported_content_types: 'Iterable[str]'):
-    """ Decorator to check if an endpoint supports the requested content type. """
+P = ParamSpec('P')
+R = TypeVar('R')
+
+
+def check_accept_header_wrapper_flask(
+        supported_content_types: 'Iterable[str]'
+) -> 'Callable[[Callable[P, R]], Callable[P, R]]':
+    """Decorator that refuses requests with an unsupported *Accept* header."""
+
+    def wrapper(
+            f: 'Callable[P, R]'
+    ) -> 'Callable[P, R]':
+        """Decorate *f* with an *Accept*-header check and return the new callable."""
 
-    def wrapper(f):
         @wraps(f)
-        def decorated(*args, **kwargs):
+        def decorated(*args: 'P.args', **kwargs: 'P.kwargs') -> 'R':
+            """Run the header check, then delegate to *f* (or return 406)."""
+
+            # 1. no Accept header → accept everything
             if not flask.request.accept_mimetypes.provided:
-                # accept anything, if Accept header is not provided
                 return f(*args, **kwargs)
 
-            for supported in supported_content_types:
-                if supported in flask.request.accept_mimetypes:
-                    return f(*args, **kwargs)
+            # 2. at least one acceptable media‑type → call the view
+            if any(s in flask.request.accept_mimetypes for s in supported_content_types):
+                return f(*args, **kwargs)
 
-            # none matched..
-            return generate_http_error_flask(
-                status_code=406,
-                exc=UnsupportedRequestedContentType.__name__,
-                exc_msg=f'The requested content type {flask.request.environ.get("HTTP_ACCEPT")} is not supported. Use {supported_content_types}.'
+            # 3. none matched → 406 response
+            return cast(
+                'R',
+                generate_http_error_flask(
+                    status_code=406,
+                    exc=UnsupportedRequestedContentType.__name__,
+                    exc_msg=(
+                        f'The requested content type '
+                        f'{flask.request.environ.get("HTTP_ACCEPT")} is not supported. '
+                        f'Use {supported_content_types}.'
+                    ),
+                ),
             )
 
         return decorated

rucio/web/rest/flaskapi/v1/config.py

@@ -281,7 +281,7 @@ class OptionSet(ErrorHandlingMethodView):
             return generate_http_error_flask(500, error, f"Could not set value '{value}' for section '{section}' option '{option}'")
 
 
-def blueprint():
+def blueprint() -> AuthenticatedBlueprint:
     bp = AuthenticatedBlueprint('config', __name__, url_prefix='/config')
 
     option_set_view = OptionSet.as_view('option_set')