rucio 32.8.6__py3-none-any.whl

rucio/core/importer.py

@@ -0,0 +1,248 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import TYPE_CHECKING
+
+from rucio.common.config import config_get
+from rucio.common.exception import RSEOperationNotSupported
+from rucio.common.types import InternalAccount
+from rucio.core import rse as rse_module, distance as distance_module, account as account_module, identity as identity_module
+from rucio.db.sqla import models
+from rucio.db.sqla.constants import RSEType, AccountType, IdentityType
+from rucio.db.sqla.session import transactional_session
+
+if TYPE_CHECKING:
+    from sqlalchemy.orm import Session
+
+
+@transactional_session
+def import_rses(rses, rse_sync_method='edit', attr_sync_method='edit', protocol_sync_method='edit', vo='def', *, session: "Session"):
+    new_rses = []
+    for rse_name in rses:
+        rse = rses[rse_name]
+        if isinstance(rse.get('rse_type'), str):
+            rse['rse_type'] = RSEType(rse['rse_type'])
+
+        if rse_module.rse_exists(rse_name, vo=vo, include_deleted=False, session=session):
+            # RSE exists and is active
+            rse_id = rse_module.get_rse_id(rse=rse_name, vo=vo, session=session)
+            selected_rse_properties = {key: rse[key] for key in rse if key in rse_module.MUTABLE_RSE_PROPERTIES}
+            rse_module.update_rse(rse_id=rse_id, parameters=selected_rse_properties, session=session)
+        elif rse_module.rse_exists(rse_name, vo=vo, include_deleted=True, session=session):
+            # RSE exists but in deleted state
+            # Should only modify the RSE if importer is configured for edit or hard sync
+            if rse_sync_method in ['edit', 'hard']:
+                rse_id = rse_module.get_rse_id(rse=rse_name, vo=vo, include_deleted=True, session=session)
+                rse_module.restore_rse(rse_id, session=session)
+                selected_rse_properties = {key: rse[key] for key in rse if key in rse_module.MUTABLE_RSE_PROPERTIES}
+                rse_module.update_rse(rse_id=rse_id, parameters=selected_rse_properties, session=session)
+            else:
+                # Config is in RSE append only mode, should not modify the disabled RSE
+                continue
+        else:
+            rse_id = rse_module.add_rse(rse=rse_name, vo=vo, deterministic=rse.get('deterministic'), volatile=rse.get('volatile'),
+                                        city=rse.get('city'), region_code=rse.get('region_code'), country_name=rse.get('country_name'),
+                                        staging_area=rse.get('staging_area'), continent=rse.get('continent'), time_zone=rse.get('time_zone'),
+                                        ISP=rse.get('ISP'), rse_type=rse.get('rse_type'), latitude=rse.get('latitude'),
+                                        longitude=rse.get('longitude'), ASN=rse.get('ASN'), availability_read=rse.get('availability_read'),
+                                        availability_write=rse.get('availability_write'), availability_delete=rse.get('availability_delete'),
+                                        session=session)
+
+        new_rses.append(rse_id)
+        # Protocols
+        new_protocols = rse.get('protocols')
+        if new_protocols:
+            # update existing, add missing and remove left over protocols
+            old_protocols = [{'scheme': protocol['scheme'], 'hostname': protocol['hostname'], 'port': protocol['port']} for protocol in rse_module.get_rse_protocols(rse_id=rse_id, session=session)['protocols']]
+            missing_protocols = [new_protocol for new_protocol in new_protocols if {'scheme': new_protocol['scheme'], 'hostname': new_protocol['hostname'], 'port': new_protocol['port']} not in old_protocols]
+            outdated_protocols = [new_protocol for new_protocol in new_protocols if {'scheme': new_protocol['scheme'], 'hostname': new_protocol['hostname'], 'port': new_protocol['port']} in old_protocols]
+            new_protocols = [{'scheme': protocol['scheme'], 'hostname': protocol['hostname'], 'port': protocol['port']} for protocol in new_protocols]
+            to_be_removed_protocols = [old_protocol for old_protocol in old_protocols if old_protocol not in new_protocols]
+
+            if protocol_sync_method == 'append':
+                outdated_protocols = []
+
+            for protocol in outdated_protocols:
+                scheme = protocol['scheme']
+                port = protocol['port']
+                hostname = protocol['hostname']
+                del protocol['scheme']
+                del protocol['hostname']
+                del protocol['port']
+                rse_module.update_protocols(rse_id=rse_id, scheme=scheme, data=protocol, hostname=hostname, port=port, session=session)
+
+            for protocol in missing_protocols:
+                rse_module.add_protocol(rse_id=rse_id, parameter=protocol, session=session)
+
+            if protocol_sync_method == 'hard':
+                for protocol in to_be_removed_protocols:
+                    scheme = protocol['scheme']
+                    port = protocol['port']
+                    hostname = protocol['hostname']
+                    rse_module.del_protocols(rse_id=rse_id, scheme=scheme, port=port, hostname=hostname, session=session)
+
+        # Limits
+        old_limits = rse_module.get_rse_limits(rse_id=rse_id, session=session)
+        for limit_name in ['MinFreeSpace']:
+            limit = rse.get(limit_name)
+            if limit:
+                if limit_name in old_limits:
+                    rse_module.delete_rse_limits(rse_id=rse_id, name=limit_name, session=session)
+                rse_module.set_rse_limits(rse_id=rse_id, name=limit_name, value=limit, session=session)
+
+        # Attributes
+        attributes = rse.get('attributes', {})
+        attributes['lfn2pfn_algorithm'] = rse.get('lfn2pfn_algorithm')
+        attributes['verify_checksum'] = rse.get('verify_checksum')
+
+        old_attributes = rse_module.list_rse_attributes(rse_id=rse_id, session=session)
+        missing_attributes = [attribute for attribute in old_attributes if attribute not in attributes]
+
+        for attr in attributes:
+            value = attributes[attr]
+            if value is not None:
+                if attr in old_attributes:
+                    if attr_sync_method not in ['append']:
+                        rse_module.del_rse_attribute(rse_id=rse_id, key=attr, session=session)
+                        rse_module.add_rse_attribute(rse_id=rse_id, key=attr, value=value, session=session)
+                else:
+                    rse_module.add_rse_attribute(rse_id=rse_id, key=attr, value=value, session=session)
+        if attr_sync_method == 'hard':
+            for attr in missing_attributes:
+                if attr != rse_name:
+                    rse_module.del_rse_attribute(rse_id=rse_id, key=attr, session=session)
+
+    # set deleted flag to RSEs that are missing in the import data
+    old_rses = [old_rse['id'] for old_rse in rse_module.list_rses(session=session)]
+    if rse_sync_method == 'hard':
+        for old_rse in old_rses:
+            if old_rse not in new_rses:
+                try:
+                    rse_module.del_rse(rse_id=old_rse, session=session)
+                except RSEOperationNotSupported:
+                    pass
+
+
+@transactional_session
+def import_distances(distances, vo='def', *, session: "Session"):
+    for src_rse_name in distances:
+        src = rse_module.get_rse_id(rse=src_rse_name, vo=vo, session=session)
+        for dest_rse_name in distances[src_rse_name]:
+            dest = rse_module.get_rse_id(rse=dest_rse_name, vo=vo, session=session)
+            distance_dict = distances[src_rse_name][dest_rse_name]
+            if 'src_rse_id' in distance_dict:
+                del distance_dict['src_rse_id']
+            if 'dest_rse_id' in distance_dict:
+                del distance_dict['dest_rse_id']
+
+            old_distance = distance_module.get_distances(src_rse_id=src, dest_rse_id=dest, session=session)
+            new_distance = distance_dict.get('distance', distance_dict.get('ranking'))
+            if old_distance:
+                distance_module.update_distances(src_rse_id=src, dest_rse_id=dest, distance=new_distance, session=session)
+            else:
+                distance_module.add_distance(src_rse_id=src, dest_rse_id=dest, distance=new_distance, session=session)
+
+
+@transactional_session
+def import_identities(identities, account_name, old_identities, old_identity_account, account_email, *, session: "Session"):
+    for identity in identities:
+        identity['type'] = IdentityType[identity['type'].upper()]
+
+    missing_identities = [identity for identity in identities if (identity['identity'], identity['type']) not in old_identities]
+    missing_identity_account = [identity for identity in identities if (identity['identity'], identity['type'], account_name) not in old_identity_account]
+    to_be_removed_identity_account = [old_identity for old_identity in old_identity_account if (old_identity[0], old_identity[1], old_identity[2]) not in
+                                      [(identity['identity'], identity['type'], account_name) for identity in identities] and old_identity[2] == account_name]
+
+    # add missing identities
+    for identity in missing_identities:
+        identity_type = identity['type']
+        password = identity.get('password')
+        identity = identity['identity']
+        if identity_type == IdentityType.USERPASS:
+            identity_module.add_identity(identity=identity, password=password, email=account_email, type_=identity_type, session=session)
+        elif identity_type == IdentityType.GSS or identity_type == IdentityType.SSH or identity_type == IdentityType.X509:
+            identity_module.add_identity(identity=identity, email=account_email, type_=identity_type, session=session)
+
+    # add missing identity-account association
+    for identity in missing_identity_account:
+        identity_module.add_account_identity(identity['identity'], identity['type'], account_name, email=account_email, session=session)
+
+    # remove identities from account-identity association
+    for identity in to_be_removed_identity_account:
+        identity_module.del_account_identity(identity=identity[0], type_=identity[1], account=identity[2], session=session)
+
+
+@transactional_session
+def import_accounts(accounts, vo='def', *, session: "Session"):
+    vo_filter = {'account': InternalAccount(account='*', vo=vo)}
+    old_accounts = {account['account']: account for account in account_module.list_accounts(filter_=vo_filter, session=session)}
+    missing_accounts = [account for account in accounts if account['account'] not in old_accounts]
+    outdated_accounts = [account for account in accounts if account['account'] in old_accounts]
+    to_be_removed_accounts = [old_account for old_account in old_accounts if old_account not in [account['account'] for account in accounts]]
+    old_identities = identity_module.list_identities(session=session)
+    old_identity_account = session.query(models.IdentityAccountAssociation.identity, models.IdentityAccountAssociation.identity_type, models.IdentityAccountAssociation.account).all()
+
+    # add missing accounts
+    for account_dict in missing_accounts:
+        account = account_dict['account']
+        email = account_dict['email']
+        account_module.add_account(account=account, type_=AccountType.USER, email=email, session=session)
+        identities = account_dict.get('identities', [])
+        if identities:
+            import_identities(identities, account, old_identities, old_identity_account, email, session=session)
+
+    # remove left over accounts
+    for account in to_be_removed_accounts:
+        if account.external != 'root':
+            account_module.del_account(account=account, session=session)
+
+    # update existing accounts
+    for account_dict in outdated_accounts:
+        account = account_dict['account']
+        email = account_dict['email']
+        old_account = old_accounts[account]
+        if email and old_account['email'] != email:
+            account_module.update_account(account, key='email', value=email, session=session)
+
+        identities = account_dict.get('identities', [])
+        if identities:
+            import_identities(identities, account, old_identities, old_identity_account, email, session=session)
+
+
+@transactional_session
+def import_data(data, vo='def', *, session: "Session"):
+    """
+    Import data to add and update records in Rucio.
+
+    :param data: data to be imported as dictionary.
+    :param session: database session in use.
+    """
+    rse_sync_method = config_get('importer', 'rse_sync_method', False, 'edit')
+    attr_sync_method = config_get('importer', 'attr_sync_method', False, 'edit')
+    protocol_sync_method = config_get('importer', 'rse_sync_method', False, 'edit')
+
+    rses = data.get('rses')
+    if rses:
+        import_rses(rses, rse_sync_method=rse_sync_method, attr_sync_method=attr_sync_method, protocol_sync_method=protocol_sync_method, vo=vo, session=session)
+
+    # Distances
+    distances = data.get('distances')
+    if distances:
+        import_distances(distances, vo=vo, session=session)
+
+    # Accounts
+    accounts = data.get('accounts')
+    if accounts:
+        import_accounts(accounts, vo=vo, session=session)

rucio/core/lifetime_exception.py

@@ -0,0 +1,377 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from configparser import NoSectionError
+from datetime import datetime, timedelta
+from re import match
+from typing import TYPE_CHECKING
+
+from sqlalchemy import or_, select, update
+from sqlalchemy.exc import IntegrityError
+from sqlalchemy.orm.exc import NoResultFound
+
+import rucio.common.policy
+from rucio.common.config import config_get, config_get_int, config_get_list
+from rucio.common.exception import RucioException, LifetimeExceptionDuplicate, LifetimeExceptionNotFound, UnsupportedOperation, ConfigNotFound
+from rucio.common.utils import generate_uuid, str_to_date
+from rucio.core.message import add_message
+from rucio.core.rse import list_rse_attributes
+from rucio.db.sqla import models
+from rucio.db.sqla.constants import DIDType, LifetimeExceptionsState
+from rucio.db.sqla.session import transactional_session, stream_session, read_session
+
+if TYPE_CHECKING:
+    from collections.abc import Iterator
+    from typing import Any, Optional, Union
+    from rucio.common.types import InternalAccount, InternalScope
+    from sqlalchemy.orm import Session
+
+
+@stream_session
+def list_exceptions(
+        exception_id: 'Optional[str]',
+        states: 'list[LifetimeExceptionsState]',
+        *,
+        session: 'Session',
+) -> 'Iterator[dict[str, Any]]':
+    """
+    List exceptions to Lifetime Model.
+
+    :param exception_id: The id of the exception
+    :param states:       The states to filter
+    :param session:      The database session in use.
+    """
+
+    state_clause = []
+    if states:
+        state_clause = [models.LifetimeExceptions.state == state for state in states]
+
+    query = select(models.LifetimeExceptions)
+    if state_clause != []:
+        query = query.where(or_(*state_clause))
+    if exception_id:
+        query = query.filter_by(id=exception_id)
+
+    for exception in session.execute(query).yield_per(5).scalars():
+        yield {'id': exception.id, 'scope': exception.scope, 'name': exception.name,
+               'did_type': exception.did_type, 'account': exception.account,
+               'pattern': exception.pattern, 'comments': exception.comments,
+               'state': exception.state, 'created_at': exception.created_at,
+               'expires_at': exception.expires_at}
+
+
+@transactional_session
+def add_exception(
+        dids: 'list[dict[str, Any]]',
+        account: 'InternalAccount',
+        pattern: 'Optional[str]',
+        comments: str,
+        expires_at: 'Optional[Union[str, datetime]]',
+        *,
+        session: 'Session'
+) -> 'dict[str, Any]':
+    """
+    Add exceptions to Lifetime Model.
+
+    :param dids:        The list of dids
+    :param account:     The account of the requester.
+    :param pattern:     The account.
+    :param comments:    The comments associated to the exception.
+    :param expires_at:  The expiration date of the exception.
+    :param session:     The database session in use.
+
+    returns:            A dictionary with id of the exceptions split by scope, datatype.
+    """
+    from rucio.core.did import get_metadata_bulk
+    result = dict()
+    result['exceptions'] = dict()
+    try:
+        max_extension = config_get_int('lifetime_model', 'max_extension', session=session)
+        if max_extension:
+            if not expires_at:
+                expires_at = datetime.utcnow() + timedelta(days=max_extension)
+            else:
+                if isinstance(expires_at, str):
+                    expires_at = str_to_date(expires_at)
+                if expires_at > datetime.utcnow() + timedelta(days=max_extension):
+                    expires_at = datetime.utcnow() + timedelta(days=max_extension)
+    except (ConfigNotFound, ValueError, NoSectionError):
+        max_extension = None
+
+    try:
+        cutoff_date = config_get('lifetime_model', 'cutoff_date', session=session)
+    except (ConfigNotFound, NoSectionError):
+        raise UnsupportedOperation('Cannot submit exception at that date.')
+    try:
+        cutoff_date = datetime.strptime(cutoff_date, '%Y-%m-%d')
+    except ValueError:
+        raise UnsupportedOperation('Cannot submit exception at that date.')
+    if cutoff_date < datetime.utcnow():
+        raise UnsupportedOperation('Cannot submit exception at that date.')
+
+    did_group = dict()
+    not_affected = list()
+    list_dids = [(did['scope'], did['name']) for did in dids]
+    metadata = [meta for meta in get_metadata_bulk(dids=dids, session=session)]
+    for did in metadata:
+        scope, name, did_type = did['scope'], did['name'], did['did_type']
+        if (scope, name) in list_dids:
+            list_dids.remove((scope, name))
+        datatype = did.get('datatype', '')
+        eol_at = did.get('eol_at', None)
+        if eol_at and eol_at < cutoff_date:
+            if (scope, datatype) not in did_group:
+                did_group[(scope, datatype)] = [list(), 0]
+            did_group[(scope, datatype)][0].append({'scope': scope, 'name': name, 'did_type': did_type})
+            did_group[(scope, datatype)][1] += did['bytes'] or 0
+        else:
+            not_affected.append((scope, name, did_type))
+    for entry in did_group:
+        exception_id = __add_exception(did_group[entry][0], account=account, pattern=pattern, comments=comments, expires_at=expires_at, estimated_volume=did_group[entry][1], session=session)
+        result['exceptions'][exception_id] = did_group[entry][0]
+    result['unknown'] = [{'scope': did[0], 'name': did[1], 'did_type': DIDType.DATASET} for did in list_dids]
+    result['not_affected'] = [{'scope': did[0], 'name': did[1], 'did_type': did[2]} for did in not_affected]
+    return result
+
+
+@transactional_session
+def __add_exception(
+        dids: 'list[dict[str, Any]]',
+        account: 'InternalAccount',
+        pattern: 'Optional[str]',
+        comments: str,
+        expires_at: 'Optional[Union[str, datetime]]',
+        estimated_volume: 'Optional[int]' = None,
+        *,
+        session: 'Session',
+) -> str:
+    """
+    Add exceptions to Lifetime Model.
+
+    :param dids:                   The list of dids
+    :param account:                The account of the requester.
+    :param pattern:                The pattern of the exception (not used).
+    :param comments:               The comments associated to the exception.
+    :param expires_at:             The expiration date of the exception.
+    :params estimated_volume:      The estimated logical volume of the exception.
+    :param session:                The database session in use.
+
+    returns:                       The id of the exception.
+    """
+    exception_id = generate_uuid()
+    text = 'Account %s requested a lifetime extension for a list of DIDs that can be found below\n' % account
+    reason = comments
+    volume = None
+    lifetime = None
+    if comments.find('||||') > -1:
+        reason, volume = comments.split('||||')
+    text += 'The reason for the extension is "%s"\n' % reason
+    text += 'It represents %s datasets\n' % len(dids)
+    if estimated_volume:
+        text += 'The estimated logical volume is %s\n' % estimated_volume
+    if volume:
+        text += 'The estimated physical volume is %s\n' % volume
+    if expires_at and isinstance(expires_at, str):
+        lifetime = str_to_date(expires_at)
+        text += 'The lifetime exception should expires on %s\n' % str(expires_at)
+    elif isinstance(expires_at, datetime):
+        lifetime = expires_at
+        text += 'The lifetime exception should expires on %s\n' % str(expires_at)
+    text += 'Link to approve or reject this request can be found at the end of the mail\n'
+    text += '\n'
+    text += 'DIDTYPE SCOPE NAME\n'
+    text += '\n'
+    truncated_message = False
+    for did in dids:
+        did_type = None
+        if 'did_type' in did:
+            if isinstance(did['did_type'], str):
+                did_type = DIDType[did['did_type']]
+            else:
+                did_type = did['did_type']
+        new_exception = models.LifetimeExceptions(id=exception_id, scope=did['scope'], name=did['name'], did_type=did_type,
+                                                  account=account, pattern=pattern, comments=reason, state=LifetimeExceptionsState.WAITING, expires_at=lifetime)
+        if len(text) < 3000:
+            text += '%s %s %s\n' % (str(did_type), did['scope'], did['name'])
+        else:
+            truncated_message = True
+        try:
+            new_exception.save(session=session, flush=False)
+        except IntegrityError as error:
+            if match('.*ORA-00001.*', str(error.args[0])) \
+                    or match('.*IntegrityError.*UNIQUE constraint failed.*', str(error.args[0])) \
+                    or match('.*1062.*Duplicate entry.*for key.*', str(error.args[0])) \
+                    or match('.*IntegrityError.*columns? .*not unique.*', error.args[0]):
+                raise LifetimeExceptionDuplicate()
+            raise RucioException(error.args[0])
+    if truncated_message:
+        text += '...\n'
+        text += 'List too long. Truncated\n'
+    text += '\n'
+    text += 'Approve:   https://rucio-ui.cern.ch/lifetime_exception?id=%s&action=approve\n' % str(exception_id)
+    text += 'Deny:      https://rucio-ui.cern.ch/lifetime_exception?id=%s&action=deny\n' % str(exception_id)
+    approvers_email = config_get_list('lifetime_model', 'approvers_email', default=[], session=session)
+
+    add_message(event_type='email',
+                payload={'body': text, 'to': approvers_email,
+                         'subject': '[RUCIO] Request to approve lifetime exception %s' % str(exception_id)},
+                session=session)
+    return exception_id
+
+
+@transactional_session
+def update_exception(
+        exception_id: str,
+        state: LifetimeExceptionsState,
+        *,
+        session: 'Session',
+) -> None:
+    """
+    Update exceptions state to Lifetime Model.
+
+    :param exception_id:   The id of the exception
+    :param state:          The states to filter
+    :param session:        The database session in use.
+    """
+    ALLOWED_STATES = (LifetimeExceptionsState.APPROVED, LifetimeExceptionsState.REJECTED)
+    if state not in ALLOWED_STATES:
+        raise UnsupportedOperation
+
+    query = update(
+        models.LifetimeExceptions
+    ).where(
+        models.LifetimeExceptions.id == exception_id
+    ).values(
+        state=state,
+        updated_at=datetime.utcnow()
+    )
+
+    if session.execute(query).rowcount == 0:
+        raise LifetimeExceptionNotFound
+
+
+@read_session
+def define_eol(
+        scope: 'InternalScope',
+        name: str,
+        rses: 'list[dict[str, Any]]',
+        *,
+        session: 'Session',
+) -> 'Optional[datetime]':
+    """
+    ATLAS policy for rules on SCRATCHDISK
+
+    :param scope:    Scope of the DID.
+    :param name:     Name of the DID.
+    :param rses:     List of RSEs.
+    :param session:  The database session in use.
+    """
+    policy = rucio.common.policy.get_policy()
+    if policy != 'atlas':
+        return None
+
+    # Check if on ATLAS managed space
+    if [rse for rse in rses if list_rse_attributes(rse_id=rse['id'], session=session).get('type') in ['LOCALGROUPDISK', 'LOCALGROUPTAPE', 'GROUPDISK', 'GROUPTAPE']]:
+        return None
+    # Now check the lifetime policy
+    try:
+        query = select(
+            models.DataIdentifier
+        ).where(
+            models.DataIdentifier.scope == scope,
+            models.DataIdentifier.name == name
+        )
+
+        did = session.execute(query).scalar_one()
+    except NoResultFound:
+        return None
+    policy_dict = rucio.common.policy.get_lifetime_policy()
+    did_type = 'other'
+    if scope.external.startswith('mc'):
+        did_type = 'mc'
+    elif scope.external.startswith('data'):
+        did_type = 'data'
+    elif scope.external.startswith('valid'):
+        did_type = 'valid'
+    else:
+        did_type = 'other'
+    for policy in policy_dict[did_type]:
+        if 'exclude' in policy:
+            to_exclude = False
+            for key in policy['exclude']:
+                meta_key = None
+                if key not in ['datatype', 'project', ]:
+                    if key == 'stream':
+                        meta_key = 'stream_name'
+                    elif key == 'tags':
+                        meta_key = 'version'
+                else:
+                    meta_key = key
+                values = policy['exclude'][key]
+                for value in values:
+                    value = value.replace('%', '.*')
+                    if meta_key and did[meta_key] and value and match(value, did[meta_key]):
+                        to_exclude = True
+                        break
+                if to_exclude:
+                    break
+            if to_exclude:
+                continue
+        if 'include' in policy:
+            match_policy = True
+            for key in policy['include']:
+                meta_key = None
+                if key not in ['datatype', 'project', ]:
+                    if key == 'stream':
+                        meta_key = 'stream_name'
+                    elif key == 'tags':
+                        meta_key = 'version'
+                    else:
+                        continue
+                else:
+                    meta_key = key
+                values = policy['include'][key]
+                to_keep = False
+                for value in values:
+                    value = value.replace('%', '.*')
+                    if meta_key and did[meta_key] and value and match(value, did[meta_key]):
+                        to_keep = True
+                        break
+                match_policy = match_policy and to_keep
+                if not to_keep:
+                    match_policy = False
+                    break
+            if match_policy:
+                if int(policy['age']) >= 12:
+                    years = int(int(policy['age']) / 12)
+                    months = int(policy['age']) - years * 12
+                    lifetime_value = 365 * years + 30 * months
+                else:
+                    lifetime_value = int(policy['age']) * 30
+                if int(policy['extension']) >= 12:
+                    years = int(int(policy['extension']) / 12)
+                    months = int(policy['extension']) - years * 12
+                    extension = 365 * years + 30 * months
+                else:
+                    extension = int(policy['extension']) * 30
+
+                default_eol_at = did.created_at + timedelta(days=lifetime_value)
+                eol_at = default_eol_at
+                if did.accessed_at:
+                    eol_at = did.accessed_at + timedelta(days=extension)
+                    if eol_at < default_eol_at:
+                        eol_at = default_eol_at
+                return eol_at
+    return None