rucio 37.0.0rc1__py3-none-any.whl

rucio/gateway/subscription.py

@@ -0,0 +1,280 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from collections import namedtuple
+from json import dumps, loads
+from typing import TYPE_CHECKING, Any, Literal, Optional, Union
+
+from rucio.common.exception import AccessDenied, InvalidObject
+from rucio.common.schema import validate_schema
+from rucio.common.types import InternalAccount, InternalScope
+from rucio.core import subscription
+from rucio.db.sqla.session import read_session, stream_session, transactional_session
+from rucio.gateway.permission import has_permission
+
+if TYPE_CHECKING:
+    from collections.abc import Iterator
+
+    from sqlalchemy.orm import Session
+
+
+SubscriptionRuleState = namedtuple('SubscriptionRuleState', ['account', 'name', 'state', 'count'])
+
+
+@transactional_session
+def add_subscription(
+    name: str,
+    account: str,
+    filter_: dict[str, Any],
+    replication_rules: list[dict[str, Any]],
+    comments: str,
+    lifetime: Union[int, Literal[False]],
+    retroactive: bool,
+    dry_run: bool,
+    priority: Optional[int] = None,
+    issuer: Optional[str] = None,
+    vo: str = 'def',
+    *,
+    session: "Session"
+) -> str:
+    """
+    Adds a new subscription which will be verified against every new added file and dataset
+
+    :param account: Account identifier
+    :param name: Name of the subscription
+    :param filter_: Dictionary of attributes by which the input data should be filtered
+                   **Example**: ``{'dsn': 'data11_hi*.express_express.*,data11_hi*physics_MinBiasOverlay*', 'account': 'tzero'}``
+    :param replication_rules: Replication rules to be set : Dictionary with keys copies, rse_expression, weight, rse_expression
+    :param comments: Comments for the subscription
+    :param lifetime: Subscription's lifetime (seconds); False if subscription has no lifetime
+    :param retroactive: Flag to know if the subscription should be applied on previous data
+    :param dry_run: Just print the subscriptions actions without actually executing them (Useful if retroactive flag is set)
+    :param priority: The priority of the subscription
+    :param issuer:  The account issuing this operation.
+    :param vo: The VO to act on.
+    :param session: The database session in use.
+    :returns: subscription_id
+    """
+    auth_result = has_permission(issuer=issuer, vo=vo, action='add_subscription', kwargs={'account': account}, session=session)
+    if not auth_result.allowed:
+        raise AccessDenied('Account %s can not add subscription. %s' % (issuer, auth_result.message))
+    try:
+        if filter_:
+            if not isinstance(filter_, dict):
+                raise TypeError('filter should be a dict')
+            validate_schema(name='subscription_filter', obj=filter_, vo=vo)
+        if replication_rules:
+            if not isinstance(replication_rules, list):
+                raise TypeError('replication_rules should be a list')
+            else:
+                for rule in replication_rules:
+                    validate_schema(name='activity', obj=rule.get('activity', 'default'), vo=vo)
+        else:
+            raise InvalidObject('You must specify a rule')
+    except ValueError as error:
+        raise TypeError(error)
+
+    internal_account = InternalAccount(account, vo=vo)
+
+    keys = ['scope', 'account']
+    types = [InternalScope, InternalAccount]
+    for _key, _type in zip(keys, types):
+        if _key in filter_:
+            if isinstance(filter_[_key], list):
+                filter_[_key] = [_type(val, vo=vo).internal for val in filter_[_key]]
+            else:
+                filter_[_key] = _type(filter_[_key], vo=vo).internal
+
+    return subscription.add_subscription(name=name, account=internal_account, filter_=dumps(filter_), replication_rules=dumps(replication_rules),
+                                         comments=comments, lifetime=lifetime, retroactive=retroactive, dry_run=dry_run, priority=priority,
+                                         session=session)
+
+
+@transactional_session
+def update_subscription(
+    name: str,
+    account: str,
+    metadata: Optional[dict[str, Any]] = None,
+    issuer: Optional[str] = None,
+    vo: str = 'def',
+    *,
+    session: "Session"
+) -> None:
+    """
+    Updates a subscription
+
+    :param name: Name of the subscription
+    :param account: Account identifier
+    :param metadata: Dictionary of metadata to update. Supported keys : filter, replication_rules, comments, lifetime, retroactive, dry_run, priority, last_processed
+    :param issuer: The account issuing this operation.
+    :param vo: The VO to act on.
+    :param session: The database session in use.
+    :raises: SubscriptionNotFound if subscription is not found
+    """
+    auth_result = has_permission(issuer=issuer, vo=vo, action='update_subscription', kwargs={'account': account}, session=session)
+    if not auth_result.allowed:
+        raise AccessDenied('Account %s can not update subscription. %s' % (issuer, auth_result.message))
+    try:
+        if not isinstance(metadata, dict):
+            raise TypeError('metadata should be a dict')
+        if 'filter' in metadata and metadata['filter']:
+            if not isinstance(metadata['filter'], dict):
+                raise TypeError('filter should be a dict')
+            validate_schema(name='subscription_filter', obj=metadata['filter'], vo=vo)
+        if 'replication_rules' in metadata and metadata['replication_rules']:
+            if not isinstance(metadata['replication_rules'], list):
+                raise TypeError('replication_rules should be a list')
+            else:
+                for rule in metadata['replication_rules']:
+                    validate_schema(name='activity', obj=rule.get('activity', 'default'), vo=vo)
+    except ValueError as error:
+        raise TypeError(error)
+
+    internal_account = InternalAccount(account, vo=vo)
+
+    if 'filter' in metadata and metadata['filter'] is not None:
+        filter_ = metadata['filter']
+        keys = ['scope', 'account']
+        types = [InternalScope, InternalAccount]
+
+        for _key, _type in zip(keys, types):
+            if _key in filter_ and filter_[_key] is not None:
+                if isinstance(filter_[_key], list):
+                    filter_[_key] = [_type(val, vo=vo).internal for val in filter_[_key]]
+                else:
+                    filter_[_key] = _type(filter_[_key], vo=vo).internal
+
+    return subscription.update_subscription(name=name, account=internal_account, metadata=metadata, session=session)
+
+
+@stream_session
+def list_subscriptions(
+    name: Optional[str] = None,
+    account: Optional[str] = None,
+    state: Optional[str] = None,
+    vo: str = 'def',
+    *,
+    session: "Session"
+) -> 'Iterator[dict[str, Any]]':
+    """
+    Returns a dictionary with the subscription information :
+    Examples: ``{'status': 'INACTIVE/ACTIVE/BROKEN', 'last_modified_date': ...}``
+
+    :param name: Name of the subscription
+    :param account: Account identifier
+    :param state: Filter for subscription state
+    :param vo: The VO to act on.
+    :param session: The database session in use.
+    :returns: Dictionary containing subscription parameter
+    :raises: exception.NotFound if subscription is not found
+    """
+
+    if account:
+        internal_account = InternalAccount(account, vo=vo)
+    else:
+        internal_account = InternalAccount('*', vo=vo)
+
+    subs = subscription.list_subscriptions(name, internal_account, state, session=session)
+
+    for sub in subs:
+        sub['account'] = sub['account'].external
+
+        if 'filter' in sub:
+            fil = loads(sub['filter'])
+            if 'account' in fil:
+                fil['account'] = [InternalAccount(acc, from_external=False).external for acc in fil['account']]
+            if 'scope' in fil:
+                fil['scope'] = [InternalScope(sco, from_external=False).external for sco in fil['scope']]
+            sub['filter'] = dumps(fil)
+
+        yield sub
+
+
+@stream_session
+def list_subscription_rule_states(
+    name: Optional[str] = None,
+    account: Optional[str] = None,
+    vo: str = 'def',
+    *,
+    session: "Session"
+) -> 'Iterator[SubscriptionRuleState]':
+    """Returns a list of with the number of rules per state for a subscription.
+
+    :param name: Name of the subscription
+    :param account: Account identifier
+    :param vo: The VO to act on.
+    :param session: The database session in use.
+    :returns: Sequence with SubscriptionRuleState named tuples (account, name, state, count)
+    """
+    if account is not None:
+        internal_account = InternalAccount(account, vo=vo)
+    else:
+        internal_account = InternalAccount('*', vo=vo)
+    subs = subscription.list_subscription_rule_states(name, internal_account, session=session)
+    for sub in subs:
+        # sub is an immutable Row so return new named tuple with edited entries
+        d = sub._asdict()
+        d['account'] = d['account'].external
+        yield SubscriptionRuleState(**d)
+
+
+@transactional_session
+def delete_subscription(
+    subscription_id: str,
+    vo: str = 'def',
+    *,
+    session: "Session"
+) -> None:
+    """
+    Deletes a subscription
+
+    :param subscription_id: Subscription identifier
+    :param vo: The VO of the user issuing command
+    :param session: The database session in use.
+    """
+
+    raise NotImplementedError
+
+
+@read_session
+def get_subscription_by_id(
+    subscription_id: str,
+    vo: str = 'def',
+    *,
+    session: "Session"
+) -> dict[str, Any]:
+    """
+    Get a specific subscription by id.
+
+    :param subscription_id: The subscription_id to select.
+    :param vo: The VO of the user issuing command.
+    :param session: The database session in use.
+
+    :raises: SubscriptionNotFound if no Subscription can be found.
+    """
+    sub = subscription.get_subscription_by_id(subscription_id, session=session)
+    if sub['account'].vo != vo:
+        raise AccessDenied('Unable to get subscription')
+
+    sub['account'] = sub['account'].external
+
+    if 'filter' in sub:
+        fil = loads(sub['filter'])
+        if 'account' in fil:
+            fil['account'] = [InternalAccount(acc, from_external=False).external for acc in fil['account']]
+        if 'scope' in fil:
+            fil['scope'] = [InternalScope(sco, from_external=False).external for sco in fil['scope']]
+        sub['filter'] = dumps(fil)
+
+    return sub

rucio/gateway/vo.py

@@ -0,0 +1,126 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import TYPE_CHECKING, Any, Optional
+
+from rucio.common import exception
+from rucio.common.schema import validate_schema
+from rucio.common.types import InternalAccount
+from rucio.core import identity
+from rucio.core import vo as vo_core
+from rucio.db.sqla.constants import IdentityType
+from rucio.db.sqla.session import read_session, transactional_session
+from rucio.gateway.permission import has_permission
+
+if TYPE_CHECKING:
+    from sqlalchemy.orm import Session
+
+
+@transactional_session
+def add_vo(new_vo: str, issuer: str, description: Optional[str] = None, email: Optional[str] = None, vo: str = 'def', *, session: "Session") -> None:
+    '''
+    Add a new VO.
+
+    :param new_vo: The name/tag of the VO to add (3 characters).
+    :param description: A description of the VO. e.g the full name or a brief description
+    :param email: A contact for the VO.
+    :param issuer: The user issuing the command.
+    :param vo: The vo of the user issuing the command.
+    :param session: The database session in use.
+    '''
+
+    new_vo = vo_core.map_vo(new_vo)
+    validate_schema('vo', new_vo, vo=vo)
+
+    kwargs = {}
+    auth_result = has_permission(issuer=issuer, action='add_vo', kwargs=kwargs, vo=vo, session=session)
+    if not auth_result.allowed:
+        raise exception.AccessDenied('Account {} cannot add a VO. {}'.format(issuer, auth_result.message))
+
+    vo_core.add_vo(vo=new_vo, description=description, email=email, session=session)
+
+
+@read_session
+def list_vos(issuer: str, vo: str = 'def', *, session: "Session") -> list[dict[str, Any]]:
+    '''
+    List the VOs.
+
+    :param issuer: The user issuing the command.
+    :param vo: The vo of the user issuing the command.
+    :param session: The database session in use.
+    '''
+    kwargs = {}
+    auth_result = has_permission(issuer=issuer, action='list_vos', kwargs=kwargs, vo=vo, session=session)
+    if not auth_result.allowed:
+        raise exception.AccessDenied('Account {} cannot list VOs. {}'.format(issuer, auth_result.message))
+
+    return vo_core.list_vos(session=session)
+
+
+@transactional_session
+def recover_vo_root_identity(
+    root_vo: str,
+    identity_key: str,
+    id_type: str,
+    email: str,
+    issuer: str,
+    default: bool = False,
+    password: Optional[str] = None,
+    vo: str = 'def',
+    *,
+    session: "Session"
+) -> None:
+    """
+    Adds a membership association between identity and the root account for given VO.
+
+    :param root_vo: The VO whose root needs recovery
+    :param identity_key: The identity key name. For example x509 DN, or a username.
+    :param id_type: The type of the authentication (x509, gss, userpass, ssh, saml).
+    :param email: The Email address associated with the identity.
+    :param issuer: The issuer account.
+    :param default: If True, the account should be used by default with the provided identity.
+    :param password: Password if id_type is userpass.
+    :param vo: the VO to act on.
+    :param session: The database session in use.
+    """
+    kwargs = {}
+    root_vo = vo_core.map_vo(root_vo)
+    auth_result = has_permission(issuer=issuer, vo=vo, action='recover_vo_root_identity', kwargs=kwargs, session=session)
+    if not auth_result.allowed:
+        raise exception.AccessDenied('Account %s can not recover root identity. %s' % (issuer, auth_result.message))
+
+    account = InternalAccount('root', vo=root_vo)
+
+    return identity.add_account_identity(identity=identity_key, type_=IdentityType[id_type.upper()], default=default,
+                                         email=email, account=account, password=password, session=session)
+
+
+@transactional_session
+def update_vo(updated_vo: str, parameters: dict[str, Any], issuer: str, vo: str = 'def', *, session: "Session") -> None:
+    """
+    Update VO properties (email, description).
+
+    :param updated_vo: The VO to update.
+    :param parameters: A dictionary with the new properties.
+    :param issuer: The user issuing the command.
+    :param vo: The VO of the user issusing the command.
+    :param session: The database session in use.
+    """
+    kwargs = {}
+    updated_vo = vo_core.map_vo(updated_vo)
+    auth_result = has_permission(issuer=issuer, action='update_vo', kwargs=kwargs, vo=vo, session=session)
+    if not auth_result.allowed:
+        raise exception.AccessDenied('Account {} cannot update VO. {}'.format(issuer, auth_result.message))
+
+    return vo_core.update_vo(vo=updated_vo, parameters=parameters, session=session)

rucio/rse/__init__.py

@@ -0,0 +1,96 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from dogpile.cache import make_region
+
+from rucio.common.client import is_client
+from rucio.rse import rsemanager
+
+if is_client():
+    setattr(rsemanager, 'CLIENT_MODE', True)
+    setattr(rsemanager, 'SERVER_MODE', False)
+else:
+    setattr(rsemanager, 'CLIENT_MODE', False)
+    setattr(rsemanager, 'SERVER_MODE', True)
+
+
+def get_rse_client(rse, vo='def', **kwarg):
+    '''
+    get_rse_client
+    '''
+    from rucio.client.rseclient import RSEClient
+    client = RSEClient(vo=vo)
+    return client.get_rse(rse)
+
+
+def get_signed_url_client(rse, service, op, url, vo='def'):
+    '''
+    get_signed_url_client
+    '''
+    from rucio.client.credentialclient import CredentialClient
+    return CredentialClient(vo=vo).get_signed_url(rse, service, op, url)
+
+
+def get_signed_url_server(rse, service, op, url, vo='def'):
+    '''
+    get_signed_url_server
+    '''
+    from rucio.core.credential import get_signed_url
+    from rucio.core.rse import get_rse_id
+
+    rse_id = get_rse_id(rse=rse, vo=vo)
+    return get_signed_url(rse_id, service, op, url)
+
+
+def rse_key_generator(namespace, fn, **kwargs):
+    '''
+    Key generator for RSE
+    '''
+    def generate_key(rse, vo='def', session=None):
+        '''
+        generate_key
+        '''
+        return '{}:{}'.format(rse, vo)
+    return generate_key
+
+
+if rsemanager.CLIENT_MODE:   # pylint:disable=no-member
+    setattr(rsemanager, '__request_rse_info', get_rse_client)
+    setattr(rsemanager, '__get_signed_url', get_signed_url_client)
+
+    # Preparing region for dogpile.cache
+    RSE_REGION = make_region(function_key_generator=rse_key_generator).configure(
+        'dogpile.cache.memory',
+        expiration_time=900)
+    setattr(rsemanager, 'RSE_REGION', RSE_REGION)
+
+
+if rsemanager.SERVER_MODE:   # pylint:disable=no-member
+    from rucio.common.cache import MemcacheRegion
+    from rucio.core.rse import get_rse_id, get_rse_protocols
+    from rucio.core.vo import map_vo
+
+    def tmp_rse_info(rse=None, vo='def', rse_id=None, session=None):
+        if rse_id is None:
+            # This can be called directly by client tools if they're co-located on a server
+            # i.e. running rucio cli on a server and during the test suite.
+            # We have to map to VO name here for this situations, despite this nominally
+            # not being a client interface.
+            rse_id = get_rse_id(rse=rse, vo=map_vo(vo))
+        return get_rse_protocols(rse_id=rse_id, session=session)
+
+    setattr(rsemanager, '__request_rse_info', tmp_rse_info)
+    setattr(rsemanager, '__get_signed_url', get_signed_url_server)
+    RSE_REGION = MemcacheRegion(expiration_time=900)
+    setattr(rsemanager, 'RSE_REGION', RSE_REGION)

rucio/rse/protocols/__init__.py

@@ -0,0 +1,13 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.