codex 1.7.15__py3-none-any.whl → 1.8.0__py3-none-any.whl

codex/librarian/scribe/importer/read/foreign_keys.py

@@ -0,0 +1,145 @@
+"""Aggregate Browser Group Trees."""
+
+from collections.abc import Mapping
+from contextlib import suppress
+
+from comicbox.identifiers import IdSources
+from comicbox.schemas.comicbox import (
+    ID_KEY_KEY,
+    ID_URL_KEY,
+    IDENTIFIERS_KEY,
+    NAME_KEY,
+    NUMBER_KEY,
+    NUMBER_TO_KEY,
+    PROTAGONIST_KEY,
+)
+from django.db.models import Field
+from django.db.models.base import Model
+
+from codex.librarian.scribe.importer.const import (
+    GROUP_MODEL_COUNT_FIELDS,
+    LINK_FKS,
+    QUERY_MODELS,
+)
+from codex.librarian.scribe.importer.query import QueryForeignKeysImporter
+from codex.librarian.scribe.importer.read.const import COMIC_FK_FIELD_NAMES_FIELD_MAP
+from codex.models.base import BaseModel
+from codex.models.groups import BrowserGroupModel, Volume
+from codex.models.identifier import Identifier, IdentifierSource
+from codex.util import max_none
+
+
+class AggregateForeignKeyMetadataImporter(QueryForeignKeysImporter):
+    """Aggregate Browser Group Trees."""
+
+    def add_query_model(
+        self,
+        model: type[Model],
+        clean_key_values: tuple,
+        clean_extra_values: frozenset | set | None = None,
+    ):
+        """Add to the queury models set for the model."""
+        if model not in self.metadata[QUERY_MODELS]:
+            self.metadata[QUERY_MODELS][model] = {}
+        if clean_key_values not in self.metadata[QUERY_MODELS][model]:
+            self.metadata[QUERY_MODELS][model][clean_key_values] = set()
+        if clean_extra_values is None:
+            clean_extra_values = frozenset()
+        else:
+            clean_extra_values = frozenset(clean_extra_values)
+        self.metadata[QUERY_MODELS][model][clean_key_values] |= clean_extra_values
+
+    def get_identifier_tuple(self, model: type[BaseModel], obj: Mapping):
+        """Parse first highest priority identifier from metadata."""
+        # Used by Objects with identifiers, not comic itself.
+        identifiers = obj.get(IDENTIFIERS_KEY)
+        if not identifiers:
+            return None
+
+        for id_source_enum in IdSources:
+            id_source = id_source_enum.value
+            if id_obj := identifiers.get(id_source):
+                break
+        else:
+            id_source, id_obj = next(identifiers.items())
+
+        if not id_obj:
+            return None
+
+        id_type = model._meta.db_table.removeprefix("codex_")
+
+        id_key = id_obj.get(ID_KEY_KEY)
+        if not id_key:
+            return None
+        id_url = id_obj.get(ID_URL_KEY)
+
+        identifier_tuple_keys = (id_source, id_type, id_key)
+        identifier_tuple_extra = frozenset([(id_url,)])
+        self.add_query_model(IdentifierSource, (id_source,))
+        self.add_query_model(Identifier, identifier_tuple_keys, identifier_tuple_extra)
+
+        return identifier_tuple_keys
+
+    def _set_simple_fk(self, related_field: Field, value) -> tuple:
+        value = related_field.get_prep_value(value)
+        if value is None:
+            return ()
+        return (value,), None
+
+    def _set_group_tree_group(
+        self,
+        model: type[BrowserGroupModel],
+        name_field: Field,
+        group: dict | None,
+        group_list: list,
+    ) -> tuple:
+        name_key = NUMBER_KEY if model == Volume else NAME_KEY
+        if group is None:
+            group = {}
+
+        group_name = group.get(name_key, model.DEFAULT_NAME)
+        clean_group_name = name_field.get_prep_value(group_name)
+        group_list.append(clean_group_name)
+        extra_vals = []
+        if model == Volume:
+            number_to = group.get(NUMBER_TO_KEY, model.DEFAULT_NAME)
+            clean_number_to = name_field.get_prep_value(number_to)
+            group_list.append(clean_number_to)
+        else:
+            identifier_tuple = self.get_identifier_tuple(model, group)
+            extra_vals.append(identifier_tuple)
+        count_key = GROUP_MODEL_COUNT_FIELDS[model]
+        if count_key:
+            count = group.get(count_key)
+            with suppress(Exception):
+                old_count_val = (
+                    self.metadata[QUERY_MODELS].get(model, {}).get(group_list)
+                )
+                count = max_none(old_count_val, count)
+            extra_vals.append(count)
+        return tuple(group_list), frozenset((tuple(extra_vals),))
+
+    def get_fk_metadata(self, md, path):
+        """Aggregate Simple Foreign Keys."""
+        group_list = []
+        for field_name, related_field in COMIC_FK_FIELD_NAMES_FIELD_MAP.items():
+            # No identifiers on many2one fks yet
+            # md_key = FIELD_NAME_TO_MD_KEY_MAP.get(field_name, field_name) if they ever diverge
+            model: type[BaseModel] = related_field.model  # pyright: ignore[reportAssignmentType], # ty: ignore[invalid-assignment]
+            md_key = field_name
+            value = md.pop(md_key, None)
+            if value is None and not issubclass(model, BrowserGroupModel):
+                continue
+
+            if issubclass(model, BrowserGroupModel):
+                key_values, extra_values = self._set_group_tree_group(
+                    model, related_field, value, group_list
+                )
+            else:
+                key_values, extra_values = self._set_simple_fk(related_field, value)
+            if not key_values and not extra_values:
+                continue
+
+            if md_key != PROTAGONIST_KEY:
+                self.add_query_model(model, key_values, extra_values)
+            self.metadata[LINK_FKS][path][field_name] = key_values

codex/librarian/scribe/importer/read/many_to_many.py

@@ -0,0 +1,237 @@
+"""Aggregate ManyToMany Metadata."""
+
+from collections.abc import Mapping
+from typing import TYPE_CHECKING
+
+from comicbox.schemas.comicbox import IDENTIFIERS_KEY, NUMBER_KEY, ROLES_KEY
+from django.db.models import CharField, Field
+from django.db.models.fields.related import ManyToManyField
+
+from codex.librarian.scribe.importer.const import (
+    COMIC_M2M_FIELDS,
+    CREDITS_FIELD_NAME,
+    IDENTIFIERS_FIELD_NAME,
+    LINK_M2MS,
+    STORY_ARC_NUMBERS_FIELD_NAME,
+    get_key_index,
+)
+from codex.librarian.scribe.importer.read.const import (
+    COMPLEX_FIELD_AGG_MAP,
+    FIELD_NAME_TO_MD_KEY_MAP,
+    ID_TYPE_KEY,
+)
+from codex.librarian.scribe.importer.read.foreign_keys import (
+    AggregateForeignKeyMetadataImporter,
+)
+from codex.models.comic import Comic
+from codex.models.groups import Folder
+from codex.models.identifier import IdentifierSource
+from codex.models.named import CreditRole
+
+if TYPE_CHECKING:
+    from codex.models.base import BaseModel
+
+
+class AggregateManyToManyMetadataImporter(AggregateForeignKeyMetadataImporter):
+    """Aggregate ManyToMany Metadata."""
+
+    def _get_m2m_metadata_dict_model_aggregate_sub_sub_value(
+        self,
+        field_name: str,
+        sub_value_obj: Mapping | None,
+        sub_sub_md_key: str,
+        sub_sub_field: Field,
+    ):
+        # sub_sub_md_key is identifiers or designation or roles or number
+        # StoryArcNumber.number can be None.
+        clean_sub_sub_values = None
+        if sub_value_obj is None:
+            return clean_sub_sub_values
+        sub_sub_value = sub_value_obj.get(sub_sub_md_key)
+        if sub_sub_value is None and field_name != STORY_ARC_NUMBERS_FIELD_NAME:
+            return clean_sub_sub_values
+
+        if sub_sub_md_key == IDENTIFIERS_KEY and sub_sub_value:
+            field = Comic._meta.get_field(field_name)
+            model: type[BaseModel] = field.related_model  # pyright: ignore[reportAssignmentType], # ty: ignore[invalid-assignment]
+            clean_sub_sub_values = self.get_identifier_tuple(model, sub_sub_value)
+        elif sub_sub_md_key == ROLES_KEY and sub_sub_value:
+            clean_sub_sub_values = set()
+            for sub_sub_sub_key_name, sub_sub_sub_value_obj in sub_sub_value.items():
+                clean_sub_sub_sub_key_name = sub_sub_field.get_prep_value(
+                    sub_sub_sub_key_name
+                )
+                sub_sub_key_identifier_tuple = self.get_identifier_tuple(
+                    CreditRole, sub_sub_sub_value_obj
+                )
+                clean_sub_sub_values.add(
+                    (clean_sub_sub_sub_key_name, sub_sub_key_identifier_tuple)
+                )
+            clean_sub_sub_values = frozenset(clean_sub_sub_values)
+        else:
+            clean_sub_sub_values = sub_sub_field.get_prep_value(sub_sub_value)
+            if sub_sub_md_key == NUMBER_KEY:
+                clean_sub_sub_values = frozenset({clean_sub_sub_values})
+        return clean_sub_sub_values
+
+    def _get_m2m_metadata_aggregate_sub_values_init(
+        self,
+        md_key: str,
+        sub_key_name_field,
+        sub_key_name: str,
+        sub_key_identifier_field,
+        sub_value_obj,
+    ):
+        name_field = (
+            sub_key_name_field
+            if isinstance(sub_key_name_field, CharField)
+            else sub_key_name_field.field
+        )
+        clean_sub_key_name = name_field.get_prep_value(sub_key_name)
+
+        clean_sub_values = []
+        sub_model = None
+        if sub_key_identifier_field:
+            sub_model = sub_key_identifier_field.field.model
+            sub_key_identifier_tuple = self.get_identifier_tuple(
+                sub_model, sub_value_obj
+            )
+            clean_sub_values.append(sub_key_identifier_tuple)
+        elif md_key == IDENTIFIERS_KEY and clean_sub_key_name:
+            sub_model = IdentifierSource
+
+        clean_sub_key = (clean_sub_key_name,)
+        if sub_model:
+            clean_sub_values_set = (
+                frozenset({tuple(clean_sub_values)}) if clean_sub_values else None
+            )
+            self.add_query_model(sub_model, clean_sub_key, clean_sub_values_set)
+        return clean_sub_key, clean_sub_values
+
+    def _get_m2m_metadata_dict_model_aggregate_sub_values(
+        self,
+        md_key: str,
+        field: ManyToManyField,
+        sub_key_name: str,
+        sub_value_obj: Mapping | None,
+    ):
+        # Clean name and if there are sub values get those.
+        # sub_key: story_arc_name_a,
+        # sub_key: character_name_a,
+        sub_key_name_field, sub_key_identifier_field, dict_field_keys = (
+            COMPLEX_FIELD_AGG_MAP[field.name]
+        )
+        clean_sub_key, clean_sub_values = (
+            self._get_m2m_metadata_aggregate_sub_values_init(
+                md_key,
+                sub_key_name_field,
+                sub_key_name,
+                sub_key_identifier_field,
+                sub_value_obj,
+            )
+        )
+
+        roles_or_numbers = set()
+        for sub_sub_md_key, sub_sub_field in dict_field_keys.items():
+            # Sub_sub_md_key is identifiers or designation or roles
+
+            if sub_sub_md_key == ID_TYPE_KEY:
+                # Special injection of identifier type
+                clean_sub_values.append(sub_sub_field)
+                continue
+
+            # Get one sub value tuple for the aggregate tuple
+            clean_sub_sub_value = (
+                self._get_m2m_metadata_dict_model_aggregate_sub_sub_value(
+                    field.name,
+                    sub_value_obj,
+                    sub_sub_md_key,
+                    sub_sub_field.field,
+                )
+            )
+            if isinstance(clean_sub_sub_value, frozenset):
+                # Special multiplier for Roles
+                roles_or_numbers |= clean_sub_sub_value
+            else:
+                clean_sub_values.append(clean_sub_sub_value)
+
+        # Create sub_map with special provisions for complex types.
+        clean_sub_map = {}
+        if roles_or_numbers:
+            clean_sub_key = clean_sub_key[0]
+            clean_sub_map = {}
+            if field.name == CREDITS_FIELD_NAME:
+                # Credits
+                for role_values in roles_or_numbers:
+                    role_keys, role_extras = role_values
+                    self.add_query_model(
+                        CreditRole, (role_keys,), frozenset({(role_extras,)})
+                    )
+                    clean_sub_map[(clean_sub_key, role_keys)] = set()
+            else:
+                # StoryArcNumbers
+                for role_values in roles_or_numbers:
+                    clean_sub_map[(clean_sub_key, role_values)] = set()
+        elif field.name == IDENTIFIERS_FIELD_NAME:
+            clean_sub_key += tuple(clean_sub_values[:2])
+            url_value = tuple(clean_sub_values[2:])
+            clean_sub_value = frozenset({url_value})
+            clean_sub_map = {clean_sub_key: clean_sub_value}
+        else:
+            clean_sub_value = (
+                frozenset((tuple(clean_sub_values),))
+                if clean_sub_values
+                else frozenset()
+            )
+            clean_sub_map = {clean_sub_key: clean_sub_value}
+        # Create query models for complex types who's keys are other types.
+        return clean_sub_map
+
+    def _get_m2m_metadata_dict_model(
+        self,
+        md_key: str,
+        field: ManyToManyField,
+        values: Mapping | list | tuple | set | frozenset,
+    ):
+        # Process values dict for a field
+        # {story_arc_name_a: { number: 1, identifiers: {} }, ...}
+        # {character_name_a: { identifiers: {} }, ...}
+        clean_values_map: dict[tuple, frozenset[tuple]] = {}
+        if not isinstance(values, Mapping):
+            values = dict.fromkeys(values)
+        related_model: type[BaseModel] = field.related_model
+        for sub_key_name, sub_value in values.items():
+            clean_sub_map = self._get_m2m_metadata_dict_model_aggregate_sub_values(
+                md_key, field, sub_key_name, sub_value
+            )
+            clean_values_map.update(clean_sub_map)
+
+        if related_model != Folder:
+            for key, value in clean_values_map.items():
+                self.add_query_model(related_model, key, value)
+        return clean_values_map
+
+    def get_m2m_metadata(self, md, path):
+        """Many_to_many fields get moved into a separate dict."""
+        m2m_md = {}
+        for field in COMIC_M2M_FIELDS:
+            md_key = FIELD_NAME_TO_MD_KEY_MAP.get(field.name, field.name)
+            values = md.pop(md_key, None)
+            if values is None:
+                continue
+            if clean_values := self._get_m2m_metadata_dict_model(md_key, field, values):
+                key_index = get_key_index(field.related_model)
+                clean_key_values = set()
+                for clean_val_tuple in clean_values:
+                    clean_key_values_tuple = clean_val_tuple[:key_index]
+                    deep_trimmed_key_values = []
+                    for val in clean_key_values_tuple:
+                        # might have to lok up index in the future but for now it's the default.
+                        deep_trimmed_val = val[0] if isinstance(val, tuple) else val
+                        deep_trimmed_key_values.append(deep_trimmed_val)
+                    clean_key_values.add(tuple(deep_trimmed_key_values))
+                if field.name not in m2m_md:
+                    m2m_md[field.name] = set()
+                m2m_md[field.name] |= clean_key_values
+
+        self.metadata[LINK_M2MS][str(path)] = m2m_md

codex/librarian/scribe/importer/search/__init__.py

@@ -0,0 +1,34 @@
+"""Sync the fts index with the imported database."""
+
+from codex.librarian.scribe.importer.search.prepare import (
+    SearchIndexPrepareImporter,
+)
+from codex.librarian.scribe.importer.statii.search import (
+    ImporterFTSCreateStatus,
+    ImporterFTSUpdateStatus,
+)
+from codex.librarian.scribe.search.handler import SearchIndexer
+from codex.librarian.scribe.search.status import SearchIndexCleanStatus
+
+_STATII = (SearchIndexCleanStatus, ImporterFTSCreateStatus, ImporterFTSUpdateStatus)
+
+
+class SearchIndexImporter(SearchIndexPrepareImporter):
+    """Sync the fts index with the imported database."""
+
+    def clean_fts(self) -> int:
+        """Clean search index of any deleted comics."""
+        indexer = SearchIndexer(
+            self.log, self.librarian_queue, self.db_write_lock, event=self.abort_event
+        )
+        return indexer.remove_stale_records(log_success=False)
+
+    def full_text_search(self):
+        """Sync the fts index with the imported database."""
+        statii = (status_class() for status_class in _STATII)
+        self.status_controller.start_many(statii)
+        try:
+            count = self.clean_fts()
+            self.import_search_index(count)
+        finally:
+            self.status_controller.finish_many(statii)

codex/librarian/scribe/importer/search/const.py

@@ -0,0 +1,50 @@
+"""Search import consts."""
+
+from types import MappingProxyType
+
+from codex.serializers.fields.browser import CountryField, LanguageField
+
+_COMICFTS_ATTRIBUTES = (
+    "collection_title",
+    "file_type",
+    "issue",
+    "name",
+    "notes",
+    "reading_direction",
+    "review",
+    "summary",
+    "updated_at",
+)
+_COMICFTS_FKS = (
+    "publisher",
+    "imprint",
+    "series",
+    "age_rating",
+    "country",
+    "language",
+    "original_format",
+    "scan_info",
+    "tagger",
+)
+_COMICFTS_M2MS = (
+    "characters",
+    "credits",
+    "genres",
+    "identifiers",
+    "locations",
+    "series_groups",
+    "sources",
+    "stories",
+    "story_arcs",
+    "tags",
+    "teams",
+    "universes",
+)
+COMICFTS_UPDATE_FIELDS = (
+    *_COMICFTS_ATTRIBUTES,
+    *_COMICFTS_FKS,
+    *_COMICFTS_M2MS,
+)
+PYCOUNTRY_FIELDS = MappingProxyType(
+    {"country": CountryField(), "language": LanguageField()}
+)

codex/librarian/scribe/importer/search/prepare.py

@@ -0,0 +1,85 @@
+"""Prepare FTS update methods used in earlier import steps."""
+
+from codex.librarian.scribe.importer.const import (
+    CREDITS_FIELD_NAME,
+    FTS_CREATE,
+    FTS_CREATED_M2MS,
+    FTS_EXISTING_M2MS,
+    FTS_UPDATE,
+    IDENTIFIERS_FIELD_NAME,
+    NON_FTS_FIELDS,
+    STORY_ARC_FIELD_NAME,
+    STORY_ARC_NUMBERS_FIELD_NAME,
+)
+from codex.librarian.scribe.importer.search.update import (
+    SearchIndexCreateUpdateImporter,
+)
+from codex.util import flatten
+
+
+class SearchIndexPrepareImporter(SearchIndexCreateUpdateImporter):
+    """Prepare FTS update methods used in earlier import steps."""
+
+    @staticmethod
+    def minify_complex_link_to_fts_tuple(field_name: str, values: tuple | frozenset):
+        """Only store the fts relevant parts of complex links."""
+        if field_name == CREDITS_FIELD_NAME:
+            values = tuple(subvalues[0] for subvalues in values)
+        if field_name == IDENTIFIERS_FIELD_NAME:
+            values = tuple(subvalues[-1] for subvalues in values)
+        elif field_name == STORY_ARC_NUMBERS_FIELD_NAME:
+            field_name = STORY_ARC_FIELD_NAME + "s"
+        return field_name, tuple(values)
+
+    @staticmethod
+    def _to_fts_tuple(values):
+        return tuple(
+            sorted(value for value in flatten(values) if isinstance(value, str))
+        )
+
+    def add_to_fts_existing(self, pk: int, field_name: str, values: tuple):
+        """Add the existing values for creating a changed search entry."""
+        if field_name in NON_FTS_FIELDS or not values:
+            return
+        if field_name == IDENTIFIERS_FIELD_NAME:
+            # sources extracton must come before identifiers is minified
+            sources = tuple(subvalues[0] for subvalues in values)
+            self.add_to_fts_existing(pk, "sources", sources)
+        field_name, values = self.minify_complex_link_to_fts_tuple(field_name, values)
+        fts_values = self._to_fts_tuple(values)
+        if not fts_values:
+            return
+        if pk not in self.metadata[FTS_EXISTING_M2MS]:
+            self.metadata[FTS_EXISTING_M2MS][pk] = {}
+        self.metadata[FTS_EXISTING_M2MS][pk][field_name] = fts_values
+
+    def add_links_to_fts(
+        self,
+        sub_key: int | str,
+        field_name: str,
+        values: tuple[str | tuple, ...],
+    ):
+        """Add a link to FTS structure."""
+        if field_name in NON_FTS_FIELDS:
+            return
+        if sub_key in self.metadata.get(FTS_UPDATE, {}):
+            key = FTS_UPDATE
+        elif sub_key in self.metadata.get(FTS_CREATE, {}):
+            key = FTS_CREATE
+        else:
+            key = FTS_UPDATE
+            if key not in self.metadata:
+                self.metadata[key] = {}
+            if sub_key not in self.metadata[key]:
+                self.metadata[key][sub_key] = {}
+            self.log.warning(
+                f"FTS import anomaly, attempting FTS update for comic {sub_key} {field_name}"
+            )
+            # Alternative might be kicking off an FTS sync
+
+        flat_values = flatten(values)
+        extra_values = (
+            self.metadata[FTS_CREATED_M2MS].get(field_name, {}).pop(flat_values, ())
+        )
+        fts_values = self._to_fts_tuple(flat_values + extra_values)
+        self.metadata[key][sub_key][field_name] = fts_values

codex/librarian/scribe/importer/search/sync_m2m.py

@@ -0,0 +1,86 @@
+"""Update fts fields for updated foreign keys with non key search values."""
+
+from django.db.models.expressions import Value
+from django.db.models.functions.datetime import Now
+from django.db.models.functions.text import Concat
+
+from codex.librarian.scribe.importer.const import FTS_UPDATED_M2MS
+from codex.librarian.scribe.importer.finish import FinishImporter
+from codex.models.comic import ComicFTS
+from codex.models.functions import GroupConcat
+
+
+class SearchIndexSyncManyToManyImporter(FinishImporter):
+    """Update fts fields for updated foreign keys with non key search values."""
+
+    @staticmethod
+    def _to_fts_str(values):
+        return ",".join(sorted(values))
+
+    @staticmethod
+    def _get_fts_m2m_concat(field_name: str):
+        rel = "comic__" + field_name
+        name_rel = rel + "__name"
+        name_concat = GroupConcat(name_rel, distinct=True)
+        if field_name == "universes":
+            exp = Concat(
+                GroupConcat(f"{rel}__designation", distinct=True),
+                Value(","),
+                name_concat,
+            )
+        else:
+            exp = name_concat
+        return exp
+
+    def _sync_fts_for_m2m_updates_model(
+        self,
+        field_name: str,
+        already_updated_comicfts_pks: tuple[int, ...],
+        update_fields: tuple[str, ...],
+        update_objs: list[ComicFTS],
+    ):
+        rel = f"comic__{field_name}__in"
+        fts_value = self._get_fts_m2m_concat(field_name)
+        model_pks = self.metadata[FTS_UPDATED_M2MS].pop(field_name)
+        self.log.debug(
+            f"Preparing {len(model_pks)} search entries for {field_name} updates."
+        )
+        comicftss = (
+            ComicFTS.objects.filter(**{rel: model_pks})
+            .exclude(pk__in=already_updated_comicfts_pks)
+            .annotate(fts_value=fts_value)
+            .only(*update_fields)
+        )
+        for comicfts in comicftss:
+            setattr(comicfts, field_name, comicfts.fts_value)  # pyright: ignore[reportAttributeAccessIssue]
+            comicfts.updated_at = Now()
+            update_objs.append(comicfts)
+
+    def sync_fts_for_m2m_updates(self, already_updated_comicfts_pks: tuple[int, ...]):
+        """Update fts entries for foreign keys."""
+        count = 0
+        update_field_names = tuple(self.metadata[FTS_UPDATED_M2MS].keys())
+
+        update_objs = []
+        for field_name in update_field_names:
+            if self.abort_event.is_set():
+                return
+            self._sync_fts_for_m2m_updates_model(
+                field_name,
+                already_updated_comicfts_pks,
+                update_field_names,
+                update_objs,
+            )
+        count += len(update_objs)
+        tags = ", ".join(update_field_names)
+        self.log.debug(
+            f"Updating {count} search index entries for comics linked to updated tags: {tags}"
+        )
+        if count:
+            ComicFTS.objects.bulk_update(update_objs, update_field_names)
+        level = "INFO" if count else "DEBUG"
+        self.log.log(
+            level,
+            f"Updated {count} search indexes entries for comics linked to updated tags: {tags}.",
+        )
+        self.metadata.pop(FTS_UPDATED_M2MS)