rucio 32.8.6__py3-none-any.whl

rucio/core/did_meta_plugins/__init__.py

@@ -0,0 +1,250 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import importlib
+from configparser import NoOptionError, NoSectionError
+from typing import TYPE_CHECKING
+
+from rucio.common import config, exception
+from rucio.db.sqla.session import read_session, transactional_session
+
+if TYPE_CHECKING:
+    from sqlalchemy.orm import Session
+
+# Set default modules.
+#
+# - "base" metadata is tied to column fields in the "dids" table. As such, types are restricted to column types and
+#   additional fields cannot be created.
+# - "custom" metadata refers to any plugin that has been written to interface with Rucio's metadata plugin system.
+#
+# By default, the default custom metadata plugin is set to the prepackaged json module. This utilises a separate table,
+# "did_meta", allowing users to add custom metadata key/value pairs of any type.
+#
+DEFAULT_BASE_METADATA_PLUGIN_MODULE_PATH = "rucio.core.did_meta_plugins.did_column_meta.DidColumnMeta"
+DEFAULT_CUSTOM_METADATA_PLUGIN_MODULE_PATH = "rucio.core.did_meta_plugins.json_meta.JSONDidMeta"
+
+# Overwrite these defaults if plugins are set in the configuration file.
+#
+if config.config_has_section('metadata'):
+    try:
+        CUSTOM_METADATA_PLUGIN_MODULE_PATHS = config.config_get('metadata', 'plugins')
+    except (NoOptionError, NoSectionError):
+        CUSTOM_METADATA_PLUGIN_MODULE_PATHS = DEFAULT_CUSTOM_METADATA_PLUGIN_MODULE_PATH
+else:
+    CUSTOM_METADATA_PLUGIN_MODULE_PATHS = DEFAULT_CUSTOM_METADATA_PLUGIN_MODULE_PATH
+METADATA_PLUGIN_MODULE_PATHS = [DEFAULT_BASE_METADATA_PLUGIN_MODULE_PATH] + CUSTOM_METADATA_PLUGIN_MODULE_PATHS.split(",")
+
+# Import plugin modules.
+#
+# Note that the order of this list is important. As the base metadata plugin module is always first, base key
+# retrieval and setting will always take precedence over custom plugins, i.e. it is not possible to set a custom key with
+# the same name as those in the base list.
+#
+# Another consequence of this is that if set_metadata() is called with multiple plugins specified, the first to return
+# True to manages_key() will be used.
+#
+METADATA_PLUGIN_MODULES = []
+for meta_module_path in METADATA_PLUGIN_MODULE_PATHS:
+    try:
+        base_module = ".".join(meta_module_path.split(".")[:-1])
+        base_class = meta_module_path.split(".")[-1]
+        metadata_plugin_module = getattr(importlib.import_module(base_module), base_class)()
+        METADATA_PLUGIN_MODULES.append(metadata_plugin_module)
+    except ImportError:
+        raise exception.PolicyPackageNotFound('Module ' + meta_module_path + ' not found')
+
+# Set restricted character set for metadata in form character: reason
+#
+RESTRICTED_CHARACTERS = {
+    '.': "Used as a delimiter for key and operator (<key>.<operator>) in filtering engine."
+}
+
+
+@read_session
+def get_metadata(scope, name, plugin="DID_COLUMN", *, session: "Session"):
+    """
+    Gets the metadata for a given did from a specified plugin.
+
+    If [plugin] is set to "all", metadata from all available metadata plugins will be returned,
+    else [plugin] can be used to only return the metadata using a specific plugin.
+
+    :param scope: The scope of the did.
+    :param name: The data identifier name.
+    :param plugin: (optional) Filter specific metadata plugins.
+    :returns: List of metadata for did.
+    :raises: NotImplementedError
+    """
+    if plugin.lower() == "all":
+        all_metadata = {}
+        for metadata_plugin in METADATA_PLUGIN_MODULES:
+            metadata = metadata_plugin.get_metadata(scope, name, session=session)
+            all_metadata.update(metadata)
+        return all_metadata
+    else:
+        for metadata_plugin in METADATA_PLUGIN_MODULES:
+            if metadata_plugin.get_plugin_name().lower() == plugin.lower():
+                return metadata_plugin.get_metadata(scope, name, session=session)
+    raise NotImplementedError('Metadata plugin "%s" is not enabled on the server.' % plugin)
+
+
+@transactional_session
+def set_metadata(scope, name, key, value, recursive=False, *, session: "Session"):
+    """
+    Sets metadata for a given did.
+
+    :param scope: The scope of the did.
+    :param name: The data identifier name.
+    :param key: Metadata key.
+    :param value: Metadata value.
+    :param recursive: (optional) Propagate the metadata change recursively to content.
+    :param session: (optional) The database session in use.
+    :raises: InvalidMetadata
+    """
+    # Check for forbidden characters in key.
+    for char in RESTRICTED_CHARACTERS:
+        if char in key:
+            raise exception.InvalidMetadata('Restricted character "{}" found in metadata key. Reason: {}'.format(
+                char,
+                RESTRICTED_CHARACTERS[char]
+            ))
+
+    # Sequentially check if each metadata plugin manages this key. Note that the order of [METADATA_PLUGIN_MODULES]
+    # means that the key is always checked for existence in the base list first.
+    metadata_was_set = False
+    for metadata_plugin in METADATA_PLUGIN_MODULES:
+        if metadata_plugin.manages_key(key, session=session):
+            metadata_plugin.set_metadata(scope, name, key, value, recursive, session=session)
+            metadata_was_set = True
+            break
+
+    if not metadata_was_set:
+        raise exception.InvalidMetadata('No plugin manages metadata key %s for DID %s:%s' % (key, scope, name))
+
+
+@transactional_session
+def set_metadata_bulk(scope, name, meta, recursive=False, *, session: "Session"):
+    """
+    Bulk sets metadata for a given did.
+
+    :param scope: The scope name.
+    :param name: The data identifier name.
+    :param meta: The key-value mapping of metadata to set.
+    :param recursive: (optional) Propagate the metadata change recursively to content.
+    :param session: (optional) The database session in use.
+    :raises: InvalidMetadata
+    """
+    metadata = meta
+
+    unmanaged_keys = list()
+    if not isinstance(metadata, dict):
+        metadata = dict(metadata)
+    metadata_plugin_keys = {metadata_plugin: [] for metadata_plugin in METADATA_PLUGIN_MODULES}
+
+    # Iterate through all keys, sequentially checking if each metadata plugin manages the considered key. If it
+    # does, add it to the list in the plugin's entry in {metadata_plugin_keys}. Note that the order of
+    # [METADATA_PLUGIN_MODULES] means that the key is always checked for existence in the base list first.
+    for key in metadata.keys():
+        # Check for forbidden characters in key.
+        for char in RESTRICTED_CHARACTERS:
+            if char in key:
+                raise exception.InvalidMetadata('Restricted character "{}" found in metadata key. Reason: {}'.format(
+                    char,
+                    RESTRICTED_CHARACTERS[char]
+                ))
+        metadata_is_included = False
+        for metadata_plugin in METADATA_PLUGIN_MODULES:
+            if metadata_plugin.manages_key(key, session=session):
+                metadata_plugin_keys[metadata_plugin].append(key)
+                metadata_is_included = True
+                break
+        if not metadata_is_included:
+            unmanaged_keys.append(key)
+    if unmanaged_keys:
+        raise exception.InvalidMetadata('No plugin manages metadata keys %s on DID %s:%s' % (unmanaged_keys, scope, name))
+
+    # For each plugin, set the metadata.
+    for metadata_plugin, keys_managed_by_this_plugin in metadata_plugin_keys.items():
+        if keys_managed_by_this_plugin:
+            this_plugin_metadata = {key: metadata[key] for key in keys_managed_by_this_plugin}
+            metadata_plugin.set_metadata_bulk(scope, name, metadata=this_plugin_metadata, recursive=recursive, session=session)
+
+
+@transactional_session
+def delete_metadata(scope, name, key, *, session: "Session"):
+    """
+    Deletes metadata stored for a given key.
+
+    :param scope: The scope of the did.
+    :param name: The name of the did.
+    :param key: Key of the metadata.
+    """
+    for metadata_plugin in METADATA_PLUGIN_MODULES:
+        if metadata_plugin.manages_key(key, session=session):
+            metadata_plugin.delete_metadata(scope, name, key, session=session)
+
+
+@read_session
+def list_dids(scope=None, filters=None, did_type='collection', ignore_case=False, limit=None,
+              offset=None, long=False, recursive=False, ignore_dids=None, *, session: "Session"):
+    """
+    Search data identifiers.
+
+    All filter keys should belong to a single plugin. Queries across plugins are not currently supported.
+
+    :param scope: the scope name.
+    :param filters: dictionary of attributes by which the results should be filtered.
+    :param did_type: the type of the did: all(container, dataset, file), collection(dataset or container), dataset, container, file.
+    :param ignore_case: ignore case distinctions.
+    :param limit: limit number.
+    :param offset: offset number.
+    :param long: Long format option to display more information for each DID.
+    :param recursive: Recursively list DIDs content.
+    :param ignore_dids: List of DIDs to refrain from yielding.
+    :param session: The database session in use.
+    :returns: List of dids satisfying metadata criteria.
+    :raises: InvalidMetadata
+    """
+    # backwards compatability for filters as single {}.
+    if isinstance(filters, dict):
+        filters = [filters]
+
+    required_unique_plugins = set()                 # keep track of which plugins are required
+    for or_group in filters:
+        for key in or_group.keys():
+            if key == 'name':                       # [name] is always passed through, and needs to be in schema of all plugins
+                continue
+            key_nooperator = key.split('.')[0]      # remove operator attribute from key if suffixed
+
+            # Iterate through the list of metadata plugins, checking which (if any) manages this particular key
+            # and appending the corresponding plugin to the set, required_unique_plugins.
+            is_this_key_managed = False
+            for metadata_plugin in METADATA_PLUGIN_MODULES:
+                if metadata_plugin.manages_key(key_nooperator, session=session):
+                    required_unique_plugins.add(metadata_plugin)
+                    is_this_key_managed = True
+                    break
+            if not is_this_key_managed:
+                raise exception.InvalidMetadata('There is no metadata plugin that manages the filter key(s) you requested.')
+
+    if not required_unique_plugins:               # if no metadata keys were specified, fall back to using the base plugin
+        required_unique_plugins = [METADATA_PLUGIN_MODULES[0]]
+    elif len(required_unique_plugins) > 1:        # check that only a single plugin is required for the query, otherwise not supported
+        raise exception.InvalidMetadata('Filter keys used do not all belong to the same metadata plugin.')
+    selected_plugin_to_use = list(required_unique_plugins)[0]
+
+    return selected_plugin_to_use.list_dids(scope=scope, filters=filters, did_type=did_type,
+                                            ignore_case=ignore_case, limit=limit,
+                                            offset=offset, long=long, recursive=recursive,
+                                            ignore_dids=ignore_dids, session=session)

rucio/core/did_meta_plugins/did_column_meta.py

@@ -0,0 +1,326 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import operator
+from datetime import datetime, timedelta
+from typing import TYPE_CHECKING
+
+from sqlalchemy import update, inspect
+from sqlalchemy.exc import CompileError, InvalidRequestError
+from sqlalchemy.orm.exc import NoResultFound
+from sqlalchemy.sql import func
+from sqlalchemy.sql.expression import true
+
+from rucio.common import exception
+from rucio.core import account_counter, rse_counter
+from rucio.core.did_meta_plugins.did_meta_plugin_interface import DidMetaPlugin
+from rucio.core.did_meta_plugins.filter_engine import FilterEngine
+from rucio.db.sqla import models
+from rucio.db.sqla.constants import DIDType
+from rucio.db.sqla.session import stream_session, read_session, transactional_session
+
+if TYPE_CHECKING:
+    from typing import Optional
+    from sqlalchemy.orm import Session
+
+
+class DidColumnMeta(DidMetaPlugin):
+    """
+    A metadata plugin to interact with the base did table metadata.
+    """
+    def __init__(self):
+        super(DidColumnMeta, self).__init__()
+        self.plugin_name = "DID_COLUMN"
+
+    @read_session
+    def get_metadata(self, scope, name, *, session: "Session"):
+        """
+        Get data identifier metadata.
+
+        :param scope: The scope name.
+        :param name: The data identifier name.
+        :param session: The database session in use.
+        """
+        try:
+            row = session.query(models.DataIdentifier).filter_by(scope=scope, name=name).\
+                with_hint(models.DataIdentifier, "INDEX(DIDS DIDS_PK)", 'oracle').one()
+            return row.to_dict()
+        except NoResultFound:
+            raise exception.DataIdentifierNotFound(f"Data identifier '{scope}:{name}' not found")
+
+    @transactional_session
+    def set_metadata(self, scope, name, key, value, recursive=False, *, session: "Session"):
+        self.set_metadata_bulk(scope=scope, name=name, metadata={key: value}, recursive=recursive, session=session)
+
+    @transactional_session
+    def set_metadata_bulk(self, scope, name, metadata, recursive=False, *, session: "Session"):
+        did_query = session.query(models.DataIdentifier).with_hint(models.DataIdentifier, "INDEX(DIDS DIDS_PK)", 'oracle').filter_by(scope=scope, name=name)
+        if did_query.one_or_none() is None:
+            raise exception.DataIdentifierNotFound("Data identifier '%s:%s' not found" % (scope, name))
+
+        remainder = {}
+        for key, value in metadata.items():
+            if key == 'eol_at' and isinstance(value, str):
+                try:
+                    eol_at = datetime.strptime(value, '%Y-%M-%d')
+                    rowcount = did_query.update({'eol_at': eol_at}, synchronize_session='fetch')
+                except TypeError as error:
+                    raise exception.InvalidValueForKey(error)
+                if not rowcount:
+                    # check for did presence
+                    raise exception.UnsupportedOperation('%s for %s:%s cannot be updated' % (key, scope, name))
+            elif key == 'lifetime':
+                try:
+                    expired_at = None
+                    if value is not None:
+                        expired_at = datetime.utcnow() + timedelta(seconds=float(value))
+                    rowcount = did_query.update({'expired_at': expired_at}, synchronize_session='fetch')
+                except TypeError as error:
+                    raise exception.InvalidValueForKey(error)
+                if not rowcount:
+                    # check for did presence
+                    raise exception.UnsupportedOperation('%s for %s:%s cannot be updated' % (key, scope, name))
+            elif key in ['guid', 'events']:
+                rowcount = did_query.filter_by(did_type=DIDType.FILE).update({key: value}, synchronize_session=False)
+                if not rowcount:
+                    # check for did presence
+                    raise exception.UnsupportedOperation('%s for %s:%s cannot be updated' % (key, scope, name))
+
+                session.query(models.DataIdentifierAssociation).filter_by(child_scope=scope, child_name=name, child_type=DIDType.FILE).update({key: value}, synchronize_session=False)
+                if key == 'events':
+                    for parent_scope, parent_name in session.query(models.DataIdentifierAssociation.scope, models.DataIdentifierAssociation.name).filter_by(child_scope=scope, child_name=name):
+                        events = session.query(func.sum(models.DataIdentifierAssociation.events)).filter_by(scope=parent_scope, name=parent_name).one()[0]
+                        session.query(models.DataIdentifier).filter_by(scope=parent_scope, name=parent_name).update({'events': events}, synchronize_session=False)
+            elif key == 'adler32':
+                rowcount = did_query.filter_by(did_type=DIDType.FILE).update({key: value}, synchronize_session=False)
+                if not rowcount:
+                    # check for did presence
+                    raise exception.UnsupportedOperation('%s for %s:%s cannot be updated' % (key, scope, name))
+
+                session.query(models.DataIdentifierAssociation).filter_by(child_scope=scope, child_name=name, child_type=DIDType.FILE).update({key: value}, synchronize_session=False)
+                session.query(models.Request).filter_by(scope=scope, name=name).update({key: value}, synchronize_session=False)
+                session.query(models.RSEFileAssociation).filter_by(scope=scope, name=name).update({key: value}, synchronize_session=False)
+            elif key == 'bytes':
+                rowcount = did_query.filter_by(did_type=DIDType.FILE).update({key: value}, synchronize_session=False)
+                if not rowcount:
+                    # check for did presence
+                    raise exception.UnsupportedOperation('%s for %s:%s cannot be updated' % (key, scope, name))
+
+                session.query(models.DataIdentifierAssociation).filter_by(child_scope=scope, child_name=name, child_type=DIDType.FILE).update({key: value}, synchronize_session=False)
+                session.query(models.Request).filter_by(scope=scope, name=name).update({key: value}, synchronize_session=False)
+
+                for account, bytes_, rse_id, rule_id in session.query(models.ReplicaLock.account, models.ReplicaLock.bytes, models.ReplicaLock.rse_id, models.ReplicaLock.rule_id).filter_by(scope=scope, name=name):
+                    session.query(models.ReplicaLock).filter_by(scope=scope, name=name, rule_id=rule_id, rse_id=rse_id).update({key: value}, synchronize_session=False)
+                    account_counter.decrease(rse_id=rse_id, account=account, files=1, bytes_=bytes_, session=session)
+                    account_counter.increase(rse_id=rse_id, account=account, files=1, bytes_=value, session=session)
+
+                for bytes_, rse_id in session.query(models.RSEFileAssociation.bytes, models.RSEFileAssociation.rse_id).filter_by(scope=scope, name=name):
+                    session.query(models.RSEFileAssociation).filter_by(scope=scope, name=name, rse_id=rse_id).update({key: value}, synchronize_session=False)
+                    rse_counter.decrease(rse_id=rse_id, files=1, bytes_=bytes_, session=session)
+                    rse_counter.increase(rse_id=rse_id, files=1, bytes_=value, session=session)
+
+                for parent_scope, parent_name in session.query(models.DataIdentifierAssociation.scope, models.DataIdentifierAssociation.name).filter_by(child_scope=scope, child_name=name):
+                    values = {}
+                    values['length'], values['bytes'], values['events'] = session.query(func.count(models.DataIdentifierAssociation.scope),
+                                                                                        func.sum(models.DataIdentifierAssociation.bytes),
+                                                                                        func.sum(models.DataIdentifierAssociation.events)).filter_by(scope=parent_scope, name=parent_name).one()
+                    session.query(models.DataIdentifier).filter_by(scope=parent_scope, name=parent_name).update(values, synchronize_session=False)
+                    session.query(models.DatasetLock).filter_by(scope=parent_scope, name=parent_name).update({'length': values['length'], 'bytes': values['bytes']}, synchronize_session=False)
+            else:
+                remainder[key] = value
+
+        if remainder:
+            try:
+                rowcount = did_query.update(remainder, synchronize_session='fetch')
+            except CompileError as error:
+                raise exception.InvalidMetadata(error)
+            except InvalidRequestError:
+                raise exception.InvalidMetadata("Some of the keys are not accepted: " + str(list(remainder.keys())))
+            if not rowcount:
+                raise exception.UnsupportedOperation('Some of the keys for %s:%s cannot be updated: %s' % (scope, name, str(list(remainder.keys()))))
+
+            # propagate metadata updates to child content
+            if recursive:
+                content_query = session.query(models.DataIdentifierAssociation.child_scope, models.DataIdentifierAssociation.child_name)
+                content_query = content_query.with_hint(models.DataIdentifierAssociation, "INDEX(CONTENTS CONTENTS_PK)", 'oracle').filter_by(scope=scope, name=name)
+
+                for child_scope, child_name in content_query:
+                    try:
+                        stmt = update(models.DataIdentifier)\
+                            .prefix_with("/*+ INDEX(DIDS DIDS_PK) */", dialect='oracle')\
+                            .filter_by(scope=child_scope, name=child_name)\
+                            .execution_options(synchronize_session='fetch')\
+                            .values(remainder)
+                        session.execute(stmt)
+                    except CompileError as error:
+                        raise exception.InvalidMetadata(error)
+                    except InvalidRequestError:
+                        raise exception.InvalidMetadata("Some of the keys are not accepted recursively: " + str(list(remainder.keys())))
+
+    @stream_session
+    def list_dids(self, scope, filters, did_type='collection', ignore_case=False, limit=None,
+                  offset=None, long=False, recursive=False, ignore_dids=None, *, session: "Session"):
+        """
+        Search data identifiers.
+
+        :param scope: the scope name.
+        :param filters: dictionary of attributes by which the results should be filtered.
+        :param did_type: the type of the did: all(container, dataset, file), collection(dataset or container), dataset, container, file.
+        :param ignore_case: ignore case distinctions.
+        :param limit: limit number.
+        :param offset: offset number.
+        :param long: Long format option to display more information for each DID.
+        :param session: The database session in use.
+        :param recursive: Recursively list DIDs content.
+        :param ignore_dids: List of DIDs to refrain from yielding.
+        """
+        if not ignore_dids:
+            ignore_dids = set()
+
+        # mapping for semantic <type> to a (set of) recognised DIDType(s).
+        type_to_did_type_mapping = {
+            'all': [DIDType.CONTAINER, DIDType.DATASET, DIDType.FILE],
+            'collection': [DIDType.CONTAINER, DIDType.DATASET],
+            'container': [DIDType.CONTAINER],
+            'dataset': [DIDType.DATASET],
+            'file': [DIDType.FILE]
+        }
+
+        # backwards compatability for filters as single {}.
+        if isinstance(filters, dict):
+            filters = [filters]
+
+        # for each or_group, make sure there is a mapped "did_type" filter.
+        # if type maps to many DIDTypes, the corresponding or_group will be copied the required number of times to satisfy all the logical possibilities.
+        filters_tmp = []
+        for or_group in filters:
+            if 'type' not in or_group:
+                or_group_type = did_type.lower()
+            else:
+                or_group_type = or_group.pop('type').lower()
+            if or_group_type not in type_to_did_type_mapping.keys():
+                raise exception.UnsupportedOperation('{} is not a valid type. Valid types are {}'.format(or_group_type, type_to_did_type_mapping.keys()))
+
+            for mapped_did_type in type_to_did_type_mapping[or_group_type]:
+                or_group['did_type'] = mapped_did_type
+                filters_tmp.append(or_group.copy())
+        filters = filters_tmp
+
+        # instantiate fe and create sqla query
+        fe = FilterEngine(filters, model_class=models.DataIdentifier)
+        query = fe.create_sqla_query(
+            additional_model_attributes=[
+                models.DataIdentifier.scope,
+                models.DataIdentifier.name,
+                models.DataIdentifier.did_type,
+                models.DataIdentifier.bytes,
+                models.DataIdentifier.length
+            ], additional_filters=[
+                (models.DataIdentifier.scope, operator.eq, scope),
+                (models.DataIdentifier.suppressed, operator.ne, true())
+            ],
+            session=session
+        )
+        query.with_hint(models.DataIdentifier, 'NO_EXPAND', 'oracle')
+
+        if limit:
+            query = query.limit(limit)
+        if recursive:
+            from rucio.core.did import list_content
+
+            # Get attached DIDs and save in list because query has to be finished before starting a new one in the recursion
+            collections_content = []
+            for did in query.yield_per(100):
+                if (did.did_type == DIDType.CONTAINER or did.did_type == DIDType.DATASET):
+                    collections_content += [d for d in list_content(scope=did.scope, name=did.name)]
+
+            # Replace any name filtering with recursed DID names.
+            for did in collections_content:
+                for or_group in filters:
+                    or_group['name'] = did['name']
+                for result in self.list_dids(scope=did['scope'], filters=filters, recursive=True, did_type=did_type, limit=limit, offset=offset,
+                                             long=long, ignore_dids=ignore_dids, session=session):
+                    yield result
+
+        for did in query.yield_per(5):                  # don't unpack this as it makes it dependent on query return order!
+            if long:
+                did_full = "{}:{}".format(did.scope, did.name)
+                if did_full not in ignore_dids:         # concatenating results of OR clauses may contain duplicate DIDs if query result sets not mutually exclusive.
+                    ignore_dids.add(did_full)
+                    yield {
+                        'scope': did.scope,
+                        'name': did.name,
+                        'did_type': str(did.did_type),
+                        'bytes': did.bytes,
+                        'length': did.length
+                    }
+            else:
+                did_full = "{}:{}".format(did.scope, did.name)
+                if did_full not in ignore_dids:         # concatenating results of OR clauses may contain duplicate DIDs if query result sets not mutually exclusive.
+                    ignore_dids.add(did_full)
+                    yield did.name
+
+    def delete_metadata(self, scope, name, key, *, session: "Optional[Session]" = None):
+        """
+        Deletes the metadata stored for the given key.
+
+        :param scope: The scope of the did.
+        :param name: The name of the did.
+        :param key: Key of the metadata.
+        """
+        raise NotImplementedError('The DidColumnMeta plugin does not currently support deleting metadata.')
+
+    def manages_key(self, key, *, session: "Optional[Session]" = None):
+        # Build list of which keys are managed by this plugin.
+        #
+        all_did_table_columns = []
+        for column in inspect(models.DataIdentifier).attrs:
+            all_did_table_columns.append(column.key)
+
+        exclude_did_table_columns = [
+            'account',
+            'availability',
+            'complete',
+            'created_at',
+            'did_type',
+            'is_open',
+            'monotonic',
+            'obsolete',
+            'scope',
+            'suppressed',
+            'updated_at'
+        ]
+
+        additional_keys = [
+            'lifetime',
+            'created_before',
+            'created_after',
+            'length.gt',
+            'length.lt',
+            'length.gte',
+            'length.lte',
+            'type'
+        ]
+
+        hardcoded_keys = list(set(all_did_table_columns) - set(exclude_did_table_columns)) + additional_keys
+
+        return key in hardcoded_keys
+
+    def get_plugin_name(self):
+        """
+        Returns a unique identifier for this plugin. This can be later used for filtering down results to this plugin only.
+        :returns: The name of the plugin.
+        """
+        return self.plugin_name