rucio 37.2.0__py3-none-any.whl → 37.4.0__py3-none-any.whl

rucio/gateway/subscription.py

@@ -20,19 +20,17 @@ from rucio.common.exception import AccessDenied, InvalidObject
 from rucio.common.schema import validate_schema
 from rucio.common.types import InternalAccount, InternalScope
 from rucio.core import subscription
-from rucio.db.sqla.session import read_session, stream_session, transactional_session
+from rucio.db.sqla.constants import DatabaseOperationType
+from rucio.db.sqla.session import db_session
 from rucio.gateway.permission import has_permission
 
 if TYPE_CHECKING:
     from collections.abc import Iterator
 
-    from sqlalchemy.orm import Session
-
 
 SubscriptionRuleState = namedtuple('SubscriptionRuleState', ['account', 'name', 'state', 'count'])
 
 
-@transactional_session
 def add_subscription(
     name: str,
     account: str,
@@ -42,11 +40,9 @@ def add_subscription(
     lifetime: Union[int, Literal[False]],
     retroactive: bool,
     dry_run: bool,
+    issuer: str,
     priority: Optional[int] = None,
-    issuer: Optional[str] = None,
     vo: str = 'def',
-    *,
-    session: "Session"
 ) -> str:
     """
     Adds a new subscription which will be verified against every new added file and dataset
@@ -63,53 +59,50 @@ def add_subscription(
     :param priority: The priority of the subscription
     :param issuer:  The account issuing this operation.
     :param vo: The VO to act on.
-    :param session: The database session in use.
     :returns: subscription_id
     """
-    auth_result = has_permission(issuer=issuer, vo=vo, action='add_subscription', kwargs={'account': account}, session=session)
-    if not auth_result.allowed:
-        raise AccessDenied('Account %s can not add subscription. %s' % (issuer, auth_result.message))
-    try:
-        if filter_:
-            if not isinstance(filter_, dict):
-                raise TypeError('filter should be a dict')
-            validate_schema(name='subscription_filter', obj=filter_, vo=vo)
-        if replication_rules:
-            if not isinstance(replication_rules, list):
-                raise TypeError('replication_rules should be a list')
-            else:
-                for rule in replication_rules:
-                    validate_schema(name='activity', obj=rule.get('activity', 'default'), vo=vo)
-        else:
-            raise InvalidObject('You must specify a rule')
-    except ValueError as error:
-        raise TypeError(error)
-
-    internal_account = InternalAccount(account, vo=vo)
-
-    keys = ['scope', 'account']
-    types = [InternalScope, InternalAccount]
-    for _key, _type in zip(keys, types):
-        if _key in filter_:
-            if isinstance(filter_[_key], list):
-                filter_[_key] = [_type(val, vo=vo).internal for val in filter_[_key]]
+    with db_session(DatabaseOperationType.WRITE) as session:
+        auth_result = has_permission(issuer=issuer, vo=vo, action='add_subscription', kwargs={'account': account}, session=session)
+        if not auth_result.allowed:
+            raise AccessDenied('Account %s can not add subscription. %s' % (issuer, auth_result.message))
+        try:
+            if filter_:
+                if not isinstance(filter_, dict):
+                    raise TypeError('filter should be a dict')
+                validate_schema(name='subscription_filter', obj=filter_, vo=vo)
+            if replication_rules:
+                if not isinstance(replication_rules, list):
+                    raise TypeError('replication_rules should be a list')
+                else:
+                    for rule in replication_rules:
+                        validate_schema(name='activity', obj=rule.get('activity', 'default'), vo=vo)
             else:
-                filter_[_key] = _type(filter_[_key], vo=vo).internal
+                raise InvalidObject('You must specify a rule')
+        except ValueError as error:
+            raise TypeError(error)
+
+        internal_account = InternalAccount(account, vo=vo)
+
+        keys = ['scope', 'account']
+        types = [InternalScope, InternalAccount]
+        for _key, _type in zip(keys, types):
+            if _key in filter_:
+                if isinstance(filter_[_key], list):
+                    filter_[_key] = [_type(val, vo=vo).internal for val in filter_[_key]]
+                else:
+                    filter_[_key] = _type(filter_[_key], vo=vo).internal
 
-    return subscription.add_subscription(name=name, account=internal_account, filter_=dumps(filter_), replication_rules=dumps(replication_rules),
-                                         comments=comments, lifetime=lifetime, retroactive=retroactive, dry_run=dry_run, priority=priority,
-                                         session=session)
+        return subscription.add_subscription(name=name, account=internal_account, filter_=dumps(filter_), replication_rules=dumps(replication_rules),
+                                             comments=comments, lifetime=lifetime, retroactive=retroactive, dry_run=dry_run, priority=priority,
+                                             session=session)
 
 
-@transactional_session
 def update_subscription(
     name: str,
     account: str,
+    issuer: str,
     metadata: Optional[dict[str, Any]] = None,
-    issuer: Optional[str] = None,
     vo: str = 'def',
-    *,
-    session: "Session"
 ) -> None:
     """
     Updates a subscription
@@ -119,53 +112,50 @@ def update_subscription(
     :param metadata: Dictionary of metadata to update. Supported keys : filter, replication_rules, comments, lifetime, retroactive, dry_run, priority, last_processed
     :param issuer: The account issuing this operation.
     :param vo: The VO to act on.
-    :param session: The database session in use.
     :raises: SubscriptionNotFound if subscription is not found
     """
-    auth_result = has_permission(issuer=issuer, vo=vo, action='update_subscription', kwargs={'account': account}, session=session)
-    if not auth_result.allowed:
-        raise AccessDenied('Account %s can not update subscription. %s' % (issuer, auth_result.message))
-    try:
-        if not isinstance(metadata, dict):
-            raise TypeError('metadata should be a dict')
-        if 'filter' in metadata and metadata['filter']:
-            if not isinstance(metadata['filter'], dict):
-                raise TypeError('filter should be a dict')
-            validate_schema(name='subscription_filter', obj=metadata['filter'], vo=vo)
-        if 'replication_rules' in metadata and metadata['replication_rules']:
-            if not isinstance(metadata['replication_rules'], list):
-                raise TypeError('replication_rules should be a list')
-            else:
-                for rule in metadata['replication_rules']:
-                    validate_schema(name='activity', obj=rule.get('activity', 'default'), vo=vo)
-    except ValueError as error:
-        raise TypeError(error)
+    with db_session(DatabaseOperationType.WRITE) as session:
+        auth_result = has_permission(issuer=issuer, vo=vo, action='update_subscription', kwargs={'account': account}, session=session)
+        if not auth_result.allowed:
+            raise AccessDenied('Account %s can not update subscription. %s' % (issuer, auth_result.message))
+        try:
+            if not isinstance(metadata, dict):
+                raise TypeError('metadata should be a dict')
+            if 'filter' in metadata and metadata['filter']:
+                if not isinstance(metadata['filter'], dict):
+                    raise TypeError('filter should be a dict')
+                validate_schema(name='subscription_filter', obj=metadata['filter'], vo=vo)
+            if 'replication_rules' in metadata and metadata['replication_rules']:
+                if not isinstance(metadata['replication_rules'], list):
+                    raise TypeError('replication_rules should be a list')
+                else:
+                    for rule in metadata['replication_rules']:
+                        validate_schema(name='activity', obj=rule.get('activity', 'default'), vo=vo)
+        except ValueError as error:
+            raise TypeError(error)
 
-    internal_account = InternalAccount(account, vo=vo)
+        internal_account = InternalAccount(account, vo=vo)
 
-    if 'filter' in metadata and metadata['filter'] is not None:
-        filter_ = metadata['filter']
-        keys = ['scope', 'account']
-        types = [InternalScope, InternalAccount]
+        if 'filter' in metadata and metadata['filter'] is not None:
+            filter_ = metadata['filter']
+            keys = ['scope', 'account']
+            types = [InternalScope, InternalAccount]
 
-        for _key, _type in zip(keys, types):
-            if _key in filter_ and filter_[_key] is not None:
-                if isinstance(filter_[_key], list):
-                    filter_[_key] = [_type(val, vo=vo).internal for val in filter_[_key]]
-                else:
-                    filter_[_key] = _type(filter_[_key], vo=vo).internal
+            for _key, _type in zip(keys, types):
+                if _key in filter_ and filter_[_key] is not None:
+                    if isinstance(filter_[_key], list):
+                        filter_[_key] = [_type(val, vo=vo).internal for val in filter_[_key]]
+                    else:
+                        filter_[_key] = _type(filter_[_key], vo=vo).internal
 
-    return subscription.update_subscription(name=name, account=internal_account, metadata=metadata, session=session)
+        return subscription.update_subscription(name=name, account=internal_account, metadata=metadata, session=session)
 
 
-@stream_session
 def list_subscriptions(
     name: Optional[str] = None,
     account: Optional[str] = None,
     state: Optional[str] = None,
     vo: str = 'def',
-    *,
-    session: "Session"
 ) -> 'Iterator[dict[str, Any]]':
     """
     Returns a dictionary with the subscription information :
@@ -175,7 +165,6 @@ def list_subscriptions(
     :param account: Account identifier
     :param state: Filter for subscription state
     :param vo: The VO to act on.
-    :param session: The database session in use.
     :returns: Dictionary containing subscription parameter
     :raises: exception.NotFound if subscription is not found
     """
@@ -185,85 +174,78 @@ def list_subscriptions(
     else:
         internal_account = InternalAccount('*', vo=vo)
 
-    subs = subscription.list_subscriptions(name, internal_account, state, session=session)
+    with db_session(DatabaseOperationType.READ) as session:
+        subs = subscription.list_subscriptions(name, internal_account, state, session=session)
 
-    for sub in subs:
-        sub['account'] = sub['account'].external
+        for sub in subs:
+            sub['account'] = sub['account'].external
 
-        if 'filter' in sub:
-            fil = loads(sub['filter'])
-            if 'account' in fil:
-                fil['account'] = [InternalAccount(acc, from_external=False).external for acc in fil['account']]
-            if 'scope' in fil:
-                fil['scope'] = [InternalScope(sco, from_external=False).external for sco in fil['scope']]
-            sub['filter'] = dumps(fil)
+            if 'filter' in sub:
+                fil = loads(sub['filter'])
+                if 'account' in fil:
+                    fil['account'] = [InternalAccount(acc, from_external=False).external for acc in fil['account']]
+                if 'scope' in fil:
+                    fil['scope'] = [InternalScope(sco, from_external=False).external for sco in fil['scope']]
+                sub['filter'] = dumps(fil)
 
-        yield sub
+            yield sub
 
 
-@stream_session
 def list_subscription_rule_states(
     name: Optional[str] = None,
     account: Optional[str] = None,
     vo: str = 'def',
-    *,
-    session: "Session"
 ) -> 'Iterator[SubscriptionRuleState]':
     """Returns a list of with the number of rules per state for a subscription.
 
     :param name: Name of the subscription
     :param account: Account identifier
     :param vo: The VO to act on.
-    :param session: The database session in use.
     :returns: Sequence with SubscriptionRuleState named tuples (account, name, state, count)
     """
     if account is not None:
         internal_account = InternalAccount(account, vo=vo)
     else:
         internal_account = InternalAccount('*', vo=vo)
-    subs = subscription.list_subscription_rule_states(name, internal_account, session=session)
-    for sub in subs:
-        # sub is an immutable Row so return new named tuple with edited entries
-        d = sub._asdict()
-        d['account'] = d['account'].external
-        yield SubscriptionRuleState(**d)
+
+    with db_session(DatabaseOperationType.READ) as session:
+        subs = subscription.list_subscription_rule_states(name, internal_account, session=session)
+
+        for sub in subs:
+            # sub is an immutable Row so return new named tuple with edited entries
+            d = sub._asdict()
+            d['account'] = d['account'].external
+            yield SubscriptionRuleState(**d)
 
 
-@transactional_session
 def delete_subscription(
     subscription_id: str,
     vo: str = 'def',
-    *,
-    session: "Session"
 ) -> None:
     """
     Deletes a subscription
 
     :param subscription_id: Subscription identifier
     :param vo: The VO of the user issuing command
-    :param session: The database session in use.
     """
 
     raise NotImplementedError
 
 
-@read_session
 def get_subscription_by_id(
     subscription_id: str,
     vo: str = 'def',
-    *,
-    session: "Session"
 ) -> dict[str, Any]:
     """
     Get a specific subscription by id.
 
     :param subscription_id: The subscription_id to select.
     :param vo: The VO of the user issuing command.
-    :param session: The database session in use.
 
     :raises: SubscriptionNotFound if no Subscription can be found.
     """
-    sub = subscription.get_subscription_by_id(subscription_id, session=session)
+    with db_session(DatabaseOperationType.READ) as session:
+        sub = subscription.get_subscription_by_id(subscription_id, session=session)
     if sub['account'].vo != vo:
         raise AccessDenied('Unable to get subscription')
 

rucio/gateway/trace.py

@@ -0,0 +1,48 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import datetime
+import json
+import uuid
+from typing import Optional
+
+from rucio.core.trace import trace as core_trace
+
+
+def trace(request: bytes, trace_ip: Optional[str]) -> None:
+    """
+    Sends the trace data to trace broker after adding additional fields
+    Adds the following fields:
+        - 'traceTimeentry': The current UTC timestamp.
+        - 'traceTimeentryUnix': The Unix timestamp with microsecond precision.
+        - 'traceIp': The client's IP address, either from 'X-Forwarded-For' header or remote address.
+        - 'traceId': A unique identifier for the trace, generated as a UUID without hyphens.
+
+    Args:
+        request_data: Request json given by client
+        trace_ip: TraceIP, either the client's IP address, or IP from "X-Forwarded-For" header
+    """
+    request_data = json.loads(request)
+    if isinstance(request_data, dict):
+        request_data = [request_data]
+
+    for item in request_data:
+        item["traceIp"] = trace_ip
+        # generate entry timestamp
+        item["traceTimeentry"] = datetime.datetime.now(datetime.timezone.utc)
+        item["traceTimeentryUnix"] = item["traceTimeentry"].timestamp()
+        # generate unique ID
+        item["traceId"] = str(uuid.uuid4()).replace("-", "").lower()
+
+        core_trace(payload=item)

rucio/vcsversion.py

@@ -4,8 +4,8 @@ This file is automatically generated; Do not edit it. :)
 '''
 VERSION_INFO = {
     'final': True,
-    'version': '37.2.0',
+    'version': '37.4.0',
     'branch_nick': 'release-37',
-    'revision_id': 'ebd2ace1ae33234044fc2d70c35f6ae8f59127ce',
-    'revno': 13666
+    'revision_id': '20eed71b1dd6d1d8e550966b7d40ea730a16c7d8',
+    'revno': 13754
 }

rucio/web/rest/flaskapi/v1/accounts.py

@@ -217,7 +217,7 @@ class Scopes(ErrorHandlingMethodView):
             description: Not acceptable
         """
         try:
-            scopes = get_scopes(account, vo=request.environ.get('vo'))
+            scopes = get_scopes(account, vo=request.environ['vo'])
         except AccountNotFound as error:
             return generate_http_error_flask(404, error)
 
@@ -264,7 +264,7 @@ class Scopes(ErrorHandlingMethodView):
             description: Scope already exists.
         """
         try:
-            add_scope(scope, account, issuer=request.environ.get('issuer'), vo=request.environ.get('vo'))
+            add_scope(scope, account, issuer=request.environ['issuer'], vo=request.environ['vo'])
         except InvalidObject as error:
             return generate_http_error_flask(400, error)
         except AccessDenied as error:

rucio/web/rest/flaskapi/v1/auth.py

@@ -916,6 +916,9 @@ class GSS(ErrorHandlingMethodView):
         appid = request.headers.get('X-Rucio-AppID', default='unknown')
         ip = request.headers.get('X-Forwarded-For', default=request.remote_addr)
 
+        if not account or not gsscred:
+            return generate_http_error_flask(400, ValueError.__name__, 'Account and REMOTE_USER must be set.')
+
         try:
             result = get_auth_token_gss(account, gsscred, appid, ip, vo=vo)
         except AccessDenied:
@@ -1212,6 +1215,9 @@ class SSH(ErrorHandlingMethodView):
         appid = request.headers.get('X-Rucio-AppID', default='unknown')
         ip = request.headers.get('X-Forwarded-For', default=request.remote_addr)
 
+        if not account or not signature:
+            return generate_http_error_flask(400, ValueError.__name__, 'Account and SSH signature must be set.')
+
         try:
             result = get_auth_token_ssh(account, signature, appid, ip, vo=vo)
         except AccessDenied:
@@ -1332,6 +1338,9 @@ class SSHChallengeToken(ErrorHandlingMethodView):
         appid = request.headers.get('X-Rucio-AppID', default='unknown')
         ip = request.headers.get('X-Forwarded-For', default=request.remote_addr)
 
+        if not account:
+            return generate_http_error_flask(400, ValueError.__name__, 'Account must be set.')
+
         result = get_ssh_challenge_token(account, appid, ip, vo=vo)
 
         if not result:
@@ -1452,6 +1461,9 @@ class SAML(ErrorHandlingMethodView):
         appid = request.headers.get('X-Rucio-AppID', default='unknown')
         ip = request.headers.get('X-Forwarded-For', default=request.remote_addr)
 
+        if not account:
+            return generate_http_error_flask(400, ValueError.__name__, 'Account must be set.')
+
         if saml_nameid:
             try:
                 result = get_auth_token_saml(account, saml_nameid, appid, ip, vo=vo)
@@ -1592,6 +1604,9 @@ class Validate(ErrorHandlingMethodView):
 
         token = request.headers.get('X-Rucio-Auth-Token', default=None)
 
+        if not token:
+            return generate_http_error_flask(400, ValueError.__name__, 'Token must be set.')
+
         result = validate_auth_token(token)
         if not result:
             return generate_http_error_flask(

rucio/web/rest/flaskapi/v1/common.py

@@ -148,6 +148,9 @@ def request_auth_env() -> Optional['ResponseReturnValue']:
 
     auth_token = flask.request.headers.get('X-Rucio-Auth-Token', default=None)
 
+    if not auth_token:
+        return generate_http_error_flask(400, ValueError.__name__, 'Token must be set.')
+
     try:
         auth = validate_auth_token(auth_token)
     except CannotAuthenticate:

rucio/web/rest/flaskapi/v1/config.py

@@ -46,9 +46,9 @@ class Config(ErrorHandlingMethodView):
             description: Not acceptable
         """
         res = {}
-        for section in config.sections(issuer=request.environ.get('issuer'), vo=request.environ.get('vo')):
+        for section in config.sections(issuer=request.environ['issuer'], vo=request.environ['vo']):
             res[section] = {}
-            for item in config.items(section, issuer=request.environ.get('issuer'), vo=request.environ.get('vo')):
+            for item in config.items(section, issuer=request.environ['issuer'], vo=request.environ['vo']):
                 res[section][item[0]] = item[1]
 
         return jsonify(res), 200
@@ -87,7 +87,7 @@ class Config(ErrorHandlingMethodView):
                 return generate_http_error_flask(400, ValueError.__name__, '')
             for option, value in section_config.items():
                 try:
-                    config.set(section=section, option=option, value=value, issuer=request.environ.get('issuer'), vo=request.environ.get('vo'))
+                    config.set(section=section, option=option, value=value, issuer=request.environ['issuer'], vo=request.environ['vo'])
                 except ConfigurationError:
                     return generate_http_error_flask(400, 'ConfigurationError', f"Could not set value '{value}' for section '{section}' option '{option}'")
         return 'Created', 201
@@ -137,7 +137,7 @@ class Section(ErrorHandlingMethodView):
             description: Not acceptable
         """
         res = {}
-        for item in config.items(section, issuer=request.environ.get('issuer'), vo=request.environ.get('vo')):
+        for item in config.items(section, issuer=request.environ['issuer'], vo=request.environ['vo']):
             res[item[0]] = item[1]
 
         if res == {}:
@@ -190,7 +190,7 @@ class OptionGetDel(ErrorHandlingMethodView):
             description: Not acceptable
         """
         try:
-            result = config.get(section=section, option=option, issuer=request.environ.get('issuer'), vo=request.environ.get('vo'))
+            result = config.get(section=section, option=option, issuer=request.environ['issuer'], vo=request.environ['vo'])
             return jsonify(result), 200
         except AccessDenied as error:
             return generate_http_error_flask(401, error, f"Access to '{section}' option '{option}' denied")
@@ -223,7 +223,7 @@ class OptionGetDel(ErrorHandlingMethodView):
           401:
             description: Invalid Auth Token
         """
-        config.remove_option(section=section, option=option, issuer=request.environ.get('issuer'), vo=request.environ.get('vo'))
+        config.remove_option(section=section, option=option, issuer=request.environ['issuer'], vo=request.environ['vo'])
         return '', 200
 
 
@@ -275,7 +275,7 @@ class OptionSet(ErrorHandlingMethodView):
                   enum: ['Could not set value {} for section {} option {}']
           """
         try:
-            config.set(section=section, option=option, value=value, issuer=request.environ.get('issuer'), vo=request.environ.get('vo'))
+            config.set(section=section, option=option, value=value, issuer=request.environ['issuer'], vo=request.environ['vo'])
             return 'Created', 201
         except ConfigurationError as error:
             return generate_http_error_flask(500, error, f"Could not set value '{value}' for section '{section}' option '{option}'")