codex 1.7.14__py3-none-any.whl → 1.8.0a2__py3-none-any.whl

codex/librarian/scribe/importer/search/prepare.py

@@ -0,0 +1,85 @@
+"""Prepare FTS update methods used in earlier import steps."""
+
+from codex.librarian.scribe.importer.const import (
+    CREDITS_FIELD_NAME,
+    FTS_CREATE,
+    FTS_CREATED_M2MS,
+    FTS_EXISTING_M2MS,
+    FTS_UPDATE,
+    IDENTIFIERS_FIELD_NAME,
+    NON_FTS_FIELDS,
+    STORY_ARC_FIELD_NAME,
+    STORY_ARC_NUMBERS_FIELD_NAME,
+)
+from codex.librarian.scribe.importer.search.update import (
+    SearchIndexCreateUpdateImporter,
+)
+from codex.util import flatten
+
+
+class SearchIndexPrepareImporter(SearchIndexCreateUpdateImporter):
+    """Prepare FTS update methods used in earlier import steps."""
+
+    @staticmethod
+    def minify_complex_link_to_fts_tuple(field_name: str, values: tuple | frozenset):
+        """Only store the fts relevant parts of complex links."""
+        if field_name == CREDITS_FIELD_NAME:
+            values = tuple(subvalues[0] for subvalues in values)
+        if field_name == IDENTIFIERS_FIELD_NAME:
+            values = tuple(subvalues[-1] for subvalues in values)
+        elif field_name == STORY_ARC_NUMBERS_FIELD_NAME:
+            field_name = STORY_ARC_FIELD_NAME + "s"
+        return field_name, tuple(values)
+
+    @staticmethod
+    def _to_fts_tuple(values):
+        return tuple(
+            sorted(value for value in flatten(values) if isinstance(value, str))
+        )
+
+    def add_to_fts_existing(self, pk: int, field_name: str, values: tuple):
+        """Add the existing values for creating a changed search entry."""
+        if field_name in NON_FTS_FIELDS or not values:
+            return
+        if field_name == IDENTIFIERS_FIELD_NAME:
+            # sources extracton must come before identifiers is minified
+            sources = tuple(subvalues[0] for subvalues in values)
+            self.add_to_fts_existing(pk, "sources", sources)
+        field_name, values = self.minify_complex_link_to_fts_tuple(field_name, values)
+        fts_values = self._to_fts_tuple(values)
+        if not fts_values:
+            return
+        if pk not in self.metadata[FTS_EXISTING_M2MS]:
+            self.metadata[FTS_EXISTING_M2MS][pk] = {}
+        self.metadata[FTS_EXISTING_M2MS][pk][field_name] = fts_values
+
+    def add_links_to_fts(
+        self,
+        sub_key: int | str,
+        field_name: str,
+        values: tuple[str | tuple, ...],
+    ):
+        """Add a link to FTS structure."""
+        if field_name in NON_FTS_FIELDS:
+            return
+        if sub_key in self.metadata.get(FTS_UPDATE, {}):
+            key = FTS_UPDATE
+        elif sub_key in self.metadata.get(FTS_CREATE, {}):
+            key = FTS_CREATE
+        else:
+            key = FTS_UPDATE
+            if key not in self.metadata:
+                self.metadata[key] = {}
+            if sub_key not in self.metadata[key]:
+                self.metadata[key][sub_key] = {}
+            self.log.warning(
+                f"FTS import anomaly, attempting FTS update for comic {sub_key} {field_name}"
+            )
+            # Alternative might be kicking off an FTS sync
+
+        flat_values = flatten(values)
+        extra_values = (
+            self.metadata[FTS_CREATED_M2MS].get(field_name, {}).pop(flat_values, ())
+        )
+        fts_values = self._to_fts_tuple(flat_values + extra_values)
+        self.metadata[key][sub_key][field_name] = fts_values

codex/librarian/scribe/importer/search/sync_m2m.py

@@ -0,0 +1,84 @@
+"""Update fts fields for updated foreign keys with non key search values."""
+
+from django.db.models.expressions import Value
+from django.db.models.functions.datetime import Now
+from django.db.models.functions.text import Concat
+
+from codex.librarian.scribe.importer.const import FTS_UPDATED_M2MS
+from codex.librarian.scribe.importer.finish import FinishImporter
+from codex.models.comic import ComicFTS
+from codex.models.functions import GroupConcat
+
+
+class SearchIndexSyncManyToManyImporter(FinishImporter):
+    """Update fts fields for updated foreign keys with non key search values."""
+
+    @staticmethod
+    def _to_fts_str(values):
+        return ",".join(sorted(values))
+
+    @staticmethod
+    def _get_fts_m2m_concat(field_name: str):
+        rel = "comic__" + field_name
+        name_rel = rel + "__name"
+        name_concat = GroupConcat(name_rel, distinct=True)
+        if field_name == "universes":
+            exp = Concat(
+                GroupConcat(f"{rel}__designation", distinct=True),
+                Value(","),
+                name_concat,
+            )
+        else:
+            exp = name_concat
+        return exp
+
+    def _sync_fts_for_m2m_updates_model(
+        self,
+        field_name: str,
+        already_updated_comicfts_pks: tuple[int, ...],
+        update_fields: tuple[str, ...],
+        update_objs: list[ComicFTS],
+    ):
+        rel = f"comic__{field_name}__in"
+        fts_value = self._get_fts_m2m_concat(field_name)
+        model_pks = self.metadata[FTS_UPDATED_M2MS].pop(field_name)
+        self.log.debug(
+            "Preparing {len(model_pks)} search entries for {field_name} updates."
+        )
+        comicftss = (
+            ComicFTS.objects.filter(**{rel: model_pks})
+            .exclude(pk__in=already_updated_comicfts_pks)
+            .annotate(fts_value=fts_value)
+            .only(*update_fields)
+        )
+        for comicfts in comicftss:
+            setattr(comicfts, field_name, comicfts.fts_value)  # pyright: ignore[reportAttributeAccessIssue]
+            comicfts.updated_at = Now()
+            update_objs.append(comicfts)
+
+    def sync_fts_for_m2m_updates(self, already_updated_comicfts_pks: tuple[int, ...]):
+        """Update fts entries for foreign keys."""
+        count = 0
+        field_names = tuple(self.metadata[FTS_UPDATED_M2MS].keys())
+
+        update_fields = field_names  # tuple(sorted(model.__name__.lower() + "s" for model in models))
+        update_objs = []
+        for field_name in field_names:
+            if self.abort_event.is_set():
+                return
+            self._sync_fts_for_m2m_updates_model(
+                field_name, already_updated_comicfts_pks, update_fields, update_objs
+            )
+        count += len(update_objs)
+        tags = ", ".join(update_fields)
+        self.log.debug(
+            f"Updating {count} search index entries for comics linked to updated tags: {tags}"
+        )
+        if count:
+            ComicFTS.objects.bulk_update(update_objs, update_fields)
+        level = "INFO" if count else "DEBUG"
+        self.log.log(
+            level,
+            f"Updated {count} search indexes entries for comics linked to updated tags: {tags}.",
+        )
+        self.metadata.pop(FTS_UPDATED_M2MS)

codex/librarian/scribe/importer/search/update.py

@@ -0,0 +1,245 @@
+"""Search Index update."""
+
+from contextlib import suppress
+from time import time
+
+from comicbox.enums.comicbox import IdSources
+from comicbox.enums.maps.identifiers import ID_SOURCE_NAME_MAP
+from django.db.models.functions.datetime import Now
+from humanize import naturaldelta
+
+from codex.librarian.scribe.importer.const import (
+    FTS_CREATE,
+    FTS_EXISTING_M2MS,
+    FTS_UPDATE,
+)
+from codex.librarian.scribe.importer.search.const import (
+    COMICFTS_UPDATE_FIELDS,
+    PYCOUNTRY_FIELDS,
+)
+from codex.librarian.scribe.importer.search.sync_m2m import (
+    SearchIndexSyncManyToManyImporter,
+)
+from codex.librarian.scribe.importer.statii.search import (
+    ImporterFTSCreateStatus,
+    ImporterFTSUpdateStatus,
+)
+from codex.librarian.status import Status
+from codex.models.comic import ComicFTS
+
+
+class SearchIndexCreateUpdateImporter(SearchIndexSyncManyToManyImporter):
+    """Search Index update methods."""
+
+    @staticmethod
+    def _get_pycountry_fts_field(entry, field_name) -> str:
+        iso_code = entry.get(field_name)
+        if not iso_code:
+            return ""
+        field = PYCOUNTRY_FIELDS[field_name]
+        return ",".join((iso_code, field.to_representation(iso_code)))
+
+    @staticmethod
+    def _get_sources_fts_field(entry) -> tuple[str, ...]:
+        sources = entry.get("sources", ())
+        names = []
+        for source_str in sources:
+            names.append(source_str)
+            with suppress(ValueError):
+                id_source = IdSources(source_str)
+                if long_name := ID_SOURCE_NAME_MAP.get(id_source):
+                    names.append(long_name)
+        return tuple(names)
+
+    @staticmethod
+    def _create_comicfts_entry_attributes(entry, *, create: bool):
+        now = Now()
+        entry["updated_at"] = now
+        if create:
+            entry["created_at"] = now
+
+    @classmethod
+    def _create_comicfts_entry_fks(cls, entry):
+        entry["country"] = cls._get_pycountry_fts_field(entry, "country")
+        entry["language"] = cls._get_pycountry_fts_field(entry, "language")
+
+    @staticmethod
+    def _to_fts_values(value):
+        """Create a tuple of fts value strings."""
+        return tuple(
+            subvalue
+            for values_tuple in value
+            for subvalue in values_tuple
+            if isinstance(subvalue, str)
+        )
+
+    def _create_comicfts_entry_m2ms(self, pk: int, entry):
+        if sources := self._get_sources_fts_field(entry):
+            entry["sources"] = sources
+        if existing_values := self.metadata[FTS_EXISTING_M2MS].get(pk):
+            for field_name in tuple(existing_values.keys()):
+                if values := existing_values.get(field_name):
+                    entry[field_name] = entry.get(field_name, ()) + values
+
+    def _create_comicfts_entry(
+        self,
+        pk,
+        entry,
+        obj_list: list[ComicFTS],
+        status: Status,
+    ):
+        self._create_comicfts_entry_m2ms(pk, entry)
+        for field_name in entry:
+            value = entry[field_name]
+            if isinstance(value, tuple):
+                entry[field_name] = ",".join(sorted(value))
+        self._create_comicfts_entry_attributes(entry, create=True)
+        self._create_comicfts_entry_fks(entry)
+        comicfts = ComicFTS(comic_id=pk, **entry)
+        obj_list.append(comicfts)
+        status.increment_complete()
+        self.status_controller.update(status)
+
+    def _update_comicfts_entry(
+        self,
+        comicfts: ComicFTS,
+        obj_list: list[ComicFTS],
+        status: Status,
+    ):
+        comic_id = comicfts.comic_id  # pyright:ignore[reportAttributeAccessIssue]
+        entry = self.metadata[FTS_UPDATE].pop(comic_id)
+        self._create_comicfts_entry_m2ms(comic_id, entry)
+        for field_name in entry:
+            value = entry[field_name]
+            if isinstance(value, tuple):
+                entry[field_name] = ",".join(sorted(value))
+        self._create_comicfts_entry_attributes(entry, create=True)
+        self._create_comicfts_entry_fks(entry)
+        for field_name, value in entry.items():
+            setattr(comicfts, field_name, value)
+        obj_list.append(comicfts)
+        status.increment_complete()
+        self.status_controller.update(status)
+
+    def _update_search_index_create_or_update(
+        self,
+        obj_list: list[ComicFTS],
+        status,
+        *,
+        create: bool,
+    ):
+        if self.abort_event.is_set():
+            return
+        verb = "create" if create else "update"
+        verbing = (verb[:-1] + "ing").capitalize()
+        num_comic_fts = len(obj_list)
+
+        batch_position = f"({status.complete}/{status.total})"
+        self.log.debug(f"{verbing} {num_comic_fts} {batch_position} search entries...")
+        if create:
+            ComicFTS.objects.bulk_create(obj_list)
+        else:
+            ComicFTS.objects.bulk_update(obj_list, COMICFTS_UPDATE_FIELDS)
+        obj_list.clear()
+
+    def _update_search_index_operate_get_status(
+        self, total_entries: int, *, create: bool
+    ):
+        status_class = ImporterFTSCreateStatus if create else ImporterFTSUpdateStatus
+        return status_class(total=total_entries)
+
+    def _update_search_index_operate(self, *, create: bool) -> tuple[int, ...]:
+        key = FTS_CREATE if create else FTS_UPDATE
+        total_entries = len(self.metadata.get(key, {}))
+        status = self._update_search_index_operate_get_status(
+            total_entries, create=create
+        )
+        updated_pks = ()
+        try:
+            verb = "create" if create else "update"
+            if not total_entries:
+                self.log.debug(f"No search entries to {verb}.")
+                return updated_pks
+            self.status_controller.start(status)
+
+            obj_list = []
+            if create:
+                entries = self.metadata.pop(FTS_CREATE, {})
+                pks = tuple(entries.keys())
+                for pk in pks:
+                    if self.abort_event.is_set():
+                        return updated_pks
+                    entry = entries.pop(pk)
+                    self._create_comicfts_entry(pk, entry, obj_list, status)
+                updated_pks = pks
+            else:
+                comic_ids = tuple(self.metadata[FTS_UPDATE].keys())
+                comicftss = ComicFTS.objects.filter(comic_id__in=comic_ids)
+                for comicfts in comicftss:
+                    if self.abort_event.is_set():
+                        return tuple(updated_pks)
+                    self._update_comicfts_entry(comicfts, obj_list, status)
+                self.metadata.pop(FTS_UPDATE)
+
+            self.log.debug(f"Preparing {total_entries} comics for search indexing...")
+            self._update_search_index_create_or_update(
+                obj_list,
+                status,
+                create=create,
+            )
+
+        finally:
+            self.status_controller.finish(status)
+        return tuple(updated_pks)
+
+    def _update_search_index_update(self):
+        """Update out of date search entries."""
+        self._update_search_index_operate(create=False)
+
+    def _update_search_index_create(self):
+        """Create missing search entries."""
+        pks = self._update_search_index_operate(create=True)
+        if self.abort_event.is_set():
+            return
+        self.sync_fts_for_m2m_updates(pks)
+
+    def _update_search_index(self, cleaned_count: int):
+        """Update or Rebuild the search index."""
+        start_time = time()
+        if self.abort_event.is_set():
+            return
+        updated_count = self._update_search_index_update()
+        if self.abort_event.is_set():
+            return
+        created_count = self._update_search_index_create()
+
+        elapsed_time = time() - start_time
+        elapsed = naturaldelta(elapsed_time)
+        cleaned = f"{cleaned_count} cleaned up" if cleaned_count else ""
+        updated = f"{updated_count} updated" if updated_count else ""
+        created = f"{created_count} created" if created_count else ""
+        summary_parts = filter(None, (cleaned, updated, created))
+        if summary := ", ".join(summary_parts):
+            level = "INFO"
+            log_txt = f"Search index entries {summary}"
+        else:
+            level = "DEBUG"
+            log_txt = "Nothing done to Search index"
+        log_txt += f" in {elapsed}."
+        self.log.log(level, log_txt)
+
+    def import_search_index(self, cleaned_count: int):
+        """Update or Rebuild the search index."""
+        self.abort_event.clear()
+        try:
+            self._update_search_index(cleaned_count)
+        except Exception:
+            self.log.exception("Update search index")
+        finally:
+            if self.abort_event.is_set():
+                self.log.info("Search Index update aborted early.")
+            self.abort_event.clear()
+            self.metadata.pop(FTS_EXISTING_M2MS)
+            self.status_controller.finish_many(
+                (ImporterFTSCreateStatus, ImporterFTSUpdateStatus)
+            )

codex/librarian/scribe/importer/statii/1

@@ -0,0 +1,27 @@
+"""Importer Link Statii."""
+
+from codex.librarian.scribe.importer.status import ImporterStatus
+
+
+class ImporterLinkStatus(ImporterStatus):
+    """Importer Link Statii."""
+
+    VERB = "Link"
+    _verbed = "Linked"
+
+
+class ImporterLinkTagsStatus(ImporterLinkStatus):
+    """Importer Link Tags Status."""
+
+    CODE = "ILT"
+    ITEM_NAME = "tags"
+
+
+class ImporterLinkCoversStatus(ImporterLinkStatus):
+    """Importer Link Covers Status."""
+
+    CODE = "ILV"
+    ITEM_NAME = "custom covers"
+
+
+LINK_STATII = (ImporterLinkTagsStatus, ImporterLinkCoversStatus)

codex/librarian/scribe/importer/statii/__init__.py

@@ -0,0 +1,19 @@
+"""Importer Statii."""
+
+from codex.librarian.scribe.importer.statii.create import CREATE_STATII
+from codex.librarian.scribe.importer.statii.delete import REMOVE_STATII
+from codex.librarian.scribe.importer.statii.link import LINK_STATII
+from codex.librarian.scribe.importer.statii.moved import MOVED_STATII
+from codex.librarian.scribe.importer.statii.query import QUERY_STATII
+from codex.librarian.scribe.importer.statii.read import READ_STATII
+from codex.librarian.scribe.importer.statii.search import IMPORTER_SEARCH_INDEX_STATII
+
+IMPORTER_STATII = (
+    *CREATE_STATII,
+    *REMOVE_STATII,
+    *LINK_STATII,
+    *MOVED_STATII,
+    *QUERY_STATII,
+    *READ_STATII,
+    *IMPORTER_SEARCH_INDEX_STATII,
+)

codex/librarian/scribe/importer/statii/create.py

@@ -0,0 +1,67 @@
+"""Importer Create Statii."""
+
+from abc import ABC
+
+from codex.librarian.scribe.importer.status import ImporterStatus
+
+
+class ImporterCreateStatus(ImporterStatus, ABC):
+    """Importer Create Statii."""
+
+
+class ImporterCreateTagsStatus(ImporterCreateStatus):
+    """Importer Create Tags Status."""
+
+    CODE = "ICT"
+    VERB = "Create"
+    ITEM_NAME = "tags"
+
+
+class ImporterUpdateTagsStatus(ImporterCreateStatus):
+    """Importer Create Tags Status."""
+
+    CODE = "IUT"
+    VERB = "Update"
+    ITEM_NAME = "tags"
+
+
+class ImporterCreateComicsStatus(ImporterCreateStatus):
+    """Importer Create Comics Status."""
+
+    CODE = "ICC"
+    VERB = "Create"
+    ITEM_NAME = "comics"
+
+
+class ImporterUpdateComicsStatus(ImporterCreateStatus):
+    """Importer Update Comics Status."""
+
+    CODE = "IUC"
+    VERB = "Update"
+    ITEM_NAME = "comics"
+
+
+class ImporterCreateCoversStatus(ImporterCreateStatus):
+    """Importer Create Tags Status."""
+
+    CODE = "ICV"
+    VERB = "Create"
+    ITEM_NAME = "custom covers"
+
+
+class ImporterUpdateCoversStatus(ImporterCreateStatus):
+    """Importer Updated Tags Status."""
+
+    CODE = "IUV"
+    VERB = "Update"
+    ITEM_NAME = "custom covers"
+
+
+CREATE_STATII = (
+    ImporterCreateTagsStatus,
+    ImporterUpdateTagsStatus,
+    ImporterCreateComicsStatus,
+    ImporterUpdateComicsStatus,
+    ImporterCreateCoversStatus,
+    ImporterUpdateCoversStatus,
+)

codex/librarian/scribe/importer/statii/delete.py

@@ -0,0 +1,39 @@
+"""Importer Remove Sattii."""
+
+from abc import ABC
+
+from codex.librarian.scribe.importer.status import ImporterStatus
+
+
+class ImporterRemoveStatus(ImporterStatus, ABC):
+    """Importer Remove Sattii."""
+
+    VERB = "Remove"
+
+
+class ImporterRemoveFoldersStatus(ImporterRemoveStatus):
+    """Importer Remove Folders Status."""
+
+    CODE = "IRF"
+    ITEM_NAME = "folders"
+
+
+class ImporterRemoveComicsStatus(ImporterRemoveStatus):
+    """Importer Remove Comics Status."""
+
+    CODE = "IRC"
+    ITEM_NAME = "comics"
+
+
+class ImporterRemoveCoversStatus(ImporterRemoveStatus):
+    """Importer Remove Covers Status."""
+
+    CODE = "IRV"
+    ITEM_NAME = "custom covers"
+
+
+REMOVE_STATII = (
+    ImporterRemoveFoldersStatus,
+    ImporterRemoveComicsStatus,
+    ImporterRemoveCoversStatus,
+)

codex/librarian/scribe/importer/statii/link.py

@@ -0,0 +1,27 @@
+"""Importer Link Statii."""
+
+from codex.librarian.scribe.importer.status import ImporterStatus
+
+
+class ImporterLinkStatus(ImporterStatus):
+    """Importer Link Statii."""
+
+    VERB = "Link"
+    _verbed = "Linked"
+
+
+class ImporterLinkTagsStatus(ImporterLinkStatus):
+    """Importer Link Tags Status."""
+
+    CODE = "ILT"
+    ITEM_NAME = "tags"
+
+
+class ImporterLinkCoversStatus(ImporterLinkStatus):
+    """Importer Link Covers Status."""
+
+    CODE = "ILV"
+    ITEM_NAME = "custom covers"
+
+
+LINK_STATII = (ImporterLinkTagsStatus, ImporterLinkCoversStatus)

codex/librarian/scribe/importer/statii/moved.py

@@ -0,0 +1,39 @@
+"""Importer Moved Statii."""
+
+from abc import ABC
+
+from codex.librarian.scribe.importer.status import ImporterStatus
+
+
+class ImporterMoveStatus(ImporterStatus, ABC):
+    """Importer Moved Status."""
+
+    VERB = "Move"
+
+
+class ImporterMoveFoldersStatus(ImporterMoveStatus):
+    """Importer Moved Folder Status."""
+
+    CODE = "IMF"
+    ITEM_NAME = "folders"
+
+
+class ImporterMoveComicsStatus(ImporterMoveStatus):
+    """Importer Moved Comics Status."""
+
+    CODE = "IMC"
+    ITEM_NAME = "comics"
+
+
+class ImporterMoveCoversStatus(ImporterMoveStatus):
+    """Importer Moved Covers Status."""
+
+    CODE = "IMV"
+    ITEM_NAME = "custom covers"
+
+
+MOVED_STATII = (
+    ImporterMoveFoldersStatus,
+    ImporterMoveComicsStatus,
+    ImporterMoveCoversStatus,
+)

codex/librarian/scribe/importer/statii/query.py

@@ -0,0 +1,46 @@
+"""Importer Query Statii."""
+
+from codex.librarian.scribe.importer.status import ImporterStatus
+
+
+class ImporterQueryStatus(ImporterStatus):
+    """Importer Query Statii."""
+
+    VERB = "Query"
+    _verbed = "Queried"
+
+
+class ImporterQueryMissingTagsStatus(ImporterQueryStatus):
+    """Importer Aggregate Status."""
+
+    CODE = "IQT"
+    ITEM_NAME = "missing tags"
+
+
+class ImporterQueryComicUpdatesStatus(ImporterQueryStatus):
+    """Importer Comic Updates Status."""
+
+    CODE = "IQC"
+    ITEM_NAME = "comics"
+
+
+class ImporterQueryTagLinksStatus(ImporterQueryStatus):
+    """Importer Tag Links Status."""
+
+    CODE = "IQL"
+    ITEM_NAME = "tag links"
+
+
+class ImporterQueryMissingCoversStatus(ImporterQueryStatus):
+    """Importer Missing Covers Status."""
+
+    CODE = "IQV"
+    ITEM_NAME = "missing custom covers"
+
+
+QUERY_STATII = (
+    ImporterQueryMissingTagsStatus,
+    ImporterQueryComicUpdatesStatus,
+    ImporterQueryTagLinksStatus,
+    ImporterQueryMissingCoversStatus,
+)