codex 1.7.15__py3-none-any.whl → 1.8.0__py3-none-any.whl

codex/librarian/scribe/search/sync.py

@@ -0,0 +1,398 @@
+"""Search Index update."""
+
+from datetime import datetime
+from time import time
+from zoneinfo import ZoneInfo
+
+from comicbox.enums.maps.identifiers import ID_SOURCE_NAME_MAP
+from django.db.models.aggregates import Max
+from django.db.models.expressions import F, Value
+from django.db.models.functions import Concat
+from django.db.models.functions.datetime import Now
+from django.db.models.query import QuerySet
+from humanize import intcomma, naturaldelta
+
+from codex.librarian.scribe.search.remove import SearchIndexerRemove
+from codex.librarian.scribe.search.status import (
+    SEARCH_INDEX_STATII,
+    SearchIndexCleanStatus,
+    SearchIndexClearStatus,
+    SearchIndexSyncCreateStatus,
+    SearchIndexSyncUpdateStatus,
+)
+from codex.librarian.status import Status
+from codex.models import Comic
+from codex.models.comic import ComicFTS
+from codex.models.functions import GroupConcat
+from codex.serializers.fields.browser import CountryField, LanguageField, PyCountryField
+from codex.settings import SEARCH_INDEX_BATCH_SIZE
+
+_COMICFTS_ATTRIBUTES = (
+    "collection_title",
+    "file_type",
+    "issue",
+    "name",
+    "notes",
+    "reading_direction",
+    "review",
+    "summary",
+    "updated_at",
+)
+_COMICFTS_FKS = (
+    "publisher",
+    "imprint",
+    "series",
+    "age_rating",
+    "country",
+    "language",
+    "original_format",
+    "scan_info",
+    "tagger",
+)
+_COMICFTS_M2MS = (
+    "characters",
+    "credits",
+    "genres",
+    "identifiers",
+    "locations",
+    "series_groups",
+    "sources",
+    "stories",
+    "story_arcs",
+    "tags",
+    "teams",
+    "universes",
+)
+_COMICFTS_UPDATE_FIELDS = (
+    *_COMICFTS_ATTRIBUTES,
+    *_COMICFTS_FKS,
+    *_COMICFTS_M2MS,
+)
+_MIN_UTC_DATE = datetime.min.replace(tzinfo=ZoneInfo("UTC"))
+_CHUNK_HUMAN_SIZE = intcomma(SEARCH_INDEX_BATCH_SIZE)
+
+
+class SearchIndexerSync(SearchIndexerRemove):
+    """Search Index update methods."""
+
+    def _init_statuses(self, rebuild):
+        """Initialize all statuses order before starting."""
+        statii: list[Status] = []
+        if rebuild:
+            statii.append(
+                SearchIndexClearStatus(),
+            )
+        else:
+            statii.extend(
+                [
+                    SearchIndexCleanStatus(),
+                    SearchIndexSyncUpdateStatus(),
+                ]
+            )
+        statii.append(SearchIndexSyncCreateStatus())
+        self.status_controller.start_many(statii)
+
+    def _update_search_index_clean(self, rebuild):
+        """Clear or clean the search index."""
+        if rebuild:
+            self.log.info("Rebuilding search index...")
+            self.clear_search_index()
+        else:
+            self.remove_stale_records(log_success=False)
+
+    @staticmethod
+    def _select_related_fts_query(qs):
+        return qs.select_related(
+            "publisher",
+            "imprint",
+            "series",
+            "country",
+            "language",
+            "scan_info",
+            "tagger",
+        )
+
+    @staticmethod
+    def _prefetch_related_fts_query(qs):
+        # Prefecthing deep relations breaks the 1000 sqlite query depth limit
+        return qs.prefetch_related(
+            "characters",
+            "credits",
+            # "credits__person",
+            "identifiers",
+            # "identifiers__source",
+            "genres",
+            "locations",
+            "series_groups",
+            "stories",
+            "story_arc_numbers",
+            # "story_arc_numbers__story_arc",
+            "tags",
+            "teams",
+            "universes",
+        )
+
+    @staticmethod
+    def _annotate_fts_query(qs):
+        return qs.annotate(
+            # Group Fks
+            fts_publisher=F("publisher__name"),
+            fts_imprint=F("imprint__name"),
+            fts_series=F("series__name"),
+            # Fks
+            fts_age_rating=F("age_rating__name"),
+            fts_country=F("country__name"),
+            fts_original_format=F("original_format__name"),
+            fts_language=F("language__name"),
+            fts_scan_info=F("scan_info__name"),
+            fts_tagger=F("tagger__name"),
+            # M2ms
+            fts_characters=GroupConcat("characters__name", distinct=True),
+            fts_credits=GroupConcat("credits__person__name", distinct=True),
+            fts_genres=GroupConcat("genres__name", distinct=True),
+            fts_identifiers=GroupConcat("identifiers__key", distinct=True),
+            fts_locations=GroupConcat("locations__name", distinct=True),
+            fts_series_groups=GroupConcat("series_groups__name", distinct=True),
+            fts_sources=GroupConcat("identifiers__source__name", distinct=True),
+            fts_stories=GroupConcat("stories__name", distinct=True),
+            fts_story_arcs=GroupConcat(
+                "story_arc_numbers__story_arc__name", distinct=True
+            ),
+            fts_tags=GroupConcat("tags__name", distinct=True),
+            fts_teams=GroupConcat("teams__name", distinct=True),
+            fts_universes=Concat(
+                GroupConcat("universes__name", distinct=True),
+                Value(","),
+                GroupConcat("universes__designation", distinct=True),
+            ),
+        )
+
+    @staticmethod
+    def _get_pycountry_fts_field(field: PyCountryField, iso_code) -> str:
+        if not iso_code:
+            return ""
+        return ",".join((iso_code, field.to_representation(iso_code)))
+
+    @staticmethod
+    def _get_sources_fts_field(sources) -> str | None:
+        names = []
+        if not sources:
+            return None
+        for source in sources.split(","):
+            names.append(source)
+            if long_name := ID_SOURCE_NAME_MAP.get(source):
+                names.append(long_name)
+        return ",".join(names)
+
+    def _update_search_index_create_or_update(
+        self,
+        obj_list: list[ComicFTS],
+        status,
+        *,
+        create: bool,
+    ):
+        if self.abort_event.is_set():
+            return
+        verb = "create" if create else "update"
+        verbing = (verb[:-1] + "ing").capitalize()
+        num_comic_fts = len(obj_list)
+
+        batch_position = f"({status.complete}/{status.total})"
+        self.log.debug(f"{verbing} {num_comic_fts} {batch_position} search entries...")
+        if create:
+            ComicFTS.objects.bulk_create(obj_list)
+        else:
+            ComicFTS.objects.bulk_update(obj_list, _COMICFTS_UPDATE_FIELDS)
+        obj_list.clear()
+
+    def _create_comicfts_entry(
+        self,
+        comic,
+        obj_list: list[ComicFTS],
+        status: Status,
+        *,
+        create: bool,
+    ):
+        country_field: PyCountryField = CountryField()  # pyright: ignore[reportAssignmentType]
+        language_field: PyCountryField = LanguageField()  # pyright: ignore[reportAssignmentType]
+
+        country = self._get_pycountry_fts_field(country_field, comic.fts_country)
+        language = self._get_pycountry_fts_field(language_field, comic.fts_language)
+        sources = self._get_sources_fts_field(comic.fts_sources)
+
+        now = Now()
+        comicfts = ComicFTS(
+            comic_id=comic.pk,
+            # Attributes
+            collection_title=comic.collection_title,
+            file_type=comic.file_type,
+            issue=comic.issue(),
+            name=comic.name,
+            notes=comic.notes,
+            reading_direction=comic.reading_direction,
+            review=comic.review,
+            summary=comic.summary,
+            updated_at=now,
+            # Group FKs
+            publisher=comic.fts_publisher,
+            imprint=comic.fts_imprint,
+            series=comic.fts_series,
+            # FKS
+            age_rating=comic.fts_age_rating,
+            country=country,
+            language=language,
+            original_format=comic.fts_original_format,
+            scan_info=comic.fts_scan_info,
+            tagger=comic.fts_tagger,
+            # ManyToMany
+            characters=comic.fts_characters,
+            credits=comic.fts_credits,
+            genres=comic.fts_genres,
+            identifiers=comic.fts_identifiers,
+            sources=sources,  # comic.fts_sources,
+            locations=comic.fts_locations,
+            series_groups=comic.fts_series_groups,
+            stories=comic.fts_stories,
+            story_arcs=comic.fts_story_arcs,
+            tags=comic.fts_tags,
+            teams=comic.fts_teams,
+            universes=comic.fts_universes,
+        )
+        if create:
+            comicfts.created_at = now
+
+        obj_list.append(comicfts)
+        status.increment_complete()
+        self.status_controller.update(status)
+
+    def _update_search_index_operate_get_status(
+        self, total_comics: int, *, create: bool
+    ):
+        status_class = (
+            SearchIndexSyncCreateStatus if create else SearchIndexSyncUpdateStatus
+        )
+        subtitle = f"Chunks of {_CHUNK_HUMAN_SIZE}" if total_comics else ""
+        return status_class(total=total_comics, subtitle=subtitle)
+
+    def _update_search_index_operate(
+        self, comics_filtered_qs: QuerySet, *, create: bool
+    ):
+        total_comics = comics_filtered_qs.count()
+
+        verb = "create" if create else "update"
+        status = self._update_search_index_operate_get_status(
+            total_comics, create=create
+        )
+
+        try:
+            if total_comics:
+                reason = (
+                    f"Batching this search engine {verb} operation in to chunks of {_CHUNK_HUMAN_SIZE}."
+                    f" Search engine {verb}s run much faster as one large batch but then there's no progress updates."
+                    " You may adjust the batch size with the environment variable CODEX_SEARCH_INDEX_BATCH_SIZE."
+                )
+            else:
+                reason = f"No search entries to {verb}."
+            self.log.debug(reason)
+            if not total_comics:
+                return total_comics
+
+            self.status_controller.start(status)
+            comics = self._select_related_fts_query(comics_filtered_qs)
+            comics = self._prefetch_related_fts_query(comics)
+            comics = self._annotate_fts_query(comics)
+            comics = comics.order_by("pk")
+
+            obj_list = []
+            prep_num = min(SEARCH_INDEX_BATCH_SIZE, total_comics)
+            self.log.debug(f"Preparing {prep_num} comics for search indexing...")
+            for comic in comics.iterator(chunk_size=SEARCH_INDEX_BATCH_SIZE):
+                if self.abort_event.is_set():
+                    break
+                self._create_comicfts_entry(comic, obj_list, status, create=create)
+                if len(obj_list) >= SEARCH_INDEX_BATCH_SIZE:
+                    self._update_search_index_create_or_update(
+                        obj_list,
+                        status,
+                        create=create,
+                    )
+                    complete = status.complete or 0
+                    if prep_num := min(
+                        SEARCH_INDEX_BATCH_SIZE, total_comics - complete
+                    ):
+                        self.log.debug(
+                            f"Preparing {prep_num} comics for search indexing..."
+                        )
+
+            self._update_search_index_create_or_update(
+                obj_list,
+                status,
+                create=create,
+            )
+
+        finally:
+            self.status_controller.finish(status)
+        return total_comics
+
+    def _update_search_index_update(self):
+        """Update out of date search entries."""
+        fts_watermark = ComicFTS.objects.aggregate(max=Max("updated_at"))["max"]
+        since = fts_watermark or "the fracturing of the multiverse"
+        self.log.info(f"Looking for search entries to update since {since}...")
+        fts_watermark = fts_watermark or _MIN_UTC_DATE
+        out_of_date_comics = (
+            Comic.objects.select_related("comicfts")
+            .exclude(comicfts__isnull=True)
+            .filter(updated_at__gt=fts_watermark)
+        )
+        return self._update_search_index_operate(out_of_date_comics, create=False)
+
+    def _update_search_index_create(self):
+        """Create missing search entries."""
+        self.log.info("Looking for missing search entries to create...")
+        missing_comics = Comic.objects.filter(comicfts__isnull=True)
+        return self._update_search_index_operate(missing_comics, create=True)
+
+    def _update_search_index(self, *, rebuild: bool):
+        """Update or Rebuild the search index."""
+        start_time = time()
+        self._init_statuses(rebuild)
+
+        if self.abort_event.is_set():
+            return
+        cleaned_count = self._update_search_index_clean(rebuild)
+        if self.abort_event.is_set():
+            return
+        updated_count = self._update_search_index_update()
+        if self.abort_event.is_set():
+            return
+        created_count = self._update_search_index_create()
+
+        elapsed_time = time() - start_time
+        elapsed = naturaldelta(elapsed_time)
+        if rebuild:
+            cleaned = "cleared entire search index"
+        elif cleaned_count:
+            cleaned = f"cleaned {cleaned_count} stale entries"
+        else:
+            cleaned = ""
+        updated = f"{updated_count} entries updated by sync" if updated_count else ""
+        created = f"{created_count} entries created by sync" if created_count else ""
+        summary_parts = filter(None, (cleaned, updated, created))
+        summary = ", ".join(summary_parts)
+        if not summary:
+            summary = "found to be already synced"
+        self.log.success(f"Search index {summary} in {elapsed}.")
+
+    def update_search_index(self, *, rebuild: bool):
+        """Update or Rebuild the search index."""
+        self.abort_event.clear()
+        try:
+            self._update_search_index(rebuild=rebuild)
+        except Exception:
+            self.log.exception("Update search index")
+        finally:
+            if self.abort_event.is_set():
+                self.log.info("Search Index update aborted early.")
+            self.abort_event.clear()
+            self.status_controller.finish_many(SEARCH_INDEX_STATII)

codex/librarian/scribe/search/tasks.py

@@ -0,0 +1,28 @@
+"""Libarian Tasks for searchd."""
+
+from dataclasses import dataclass
+
+from codex.librarian.scribe.tasks import ScribeTask
+
+
+class SearchIndexerTask(ScribeTask):
+    """Tasks for the search indexer."""
+
+
+@dataclass
+class SearchIndexSyncTask(SearchIndexerTask):
+    """Update the search index."""
+
+    rebuild: bool = False
+
+
+class SearchIndexOptimizeTask(SearchIndexerTask):
+    """Optimize search index."""
+
+
+class SearchIndexCleanStaleTask(SearchIndexerTask):
+    """Remove stale records."""
+
+
+class SearchIndexClearTask(SearchIndexerTask):
+    """Clear current search index."""

codex/librarian/scribe/status.py

@@ -0,0 +1,21 @@
+"""Librarian Status for scribe bulk writes."""
+
+from abc import ABC
+
+from codex.librarian.status import Status
+
+
+class ScribeStatus(Status, ABC):
+    """Scribe Statii."""
+
+
+class UpdateGroupTimestampsStatus(ScribeStatus):
+    """Update Group Timestamps Status."""
+
+    CODE = "IGU"
+    ITEM_NAME = "browser groups"
+    VERB = "Update timestamps for"
+    _verbed = "Updated timestamps for"
+
+
+SCRIBE_STATII = (UpdateGroupTimestampsStatus,)

codex/librarian/scribe/tasks.py

@@ -0,0 +1,32 @@
+"""DB Import Tasks."""
+
+from abc import ABC
+from dataclasses import dataclass
+from datetime import datetime
+
+
+class ScribeTask(ABC):  # noqa: B024
+    """Tasks for scribed."""
+
+
+@dataclass
+class UpdateGroupsTask(ScribeTask):
+    """Force the update of group timestamp."""
+
+    start_time: datetime | None = None
+
+
+@dataclass
+class LazyImportComicsTask(ScribeTask):
+    """Lazy import of metadaa for existing comics."""
+
+    group: str
+    pks: frozenset[int]
+
+
+class ImportAbortTask(ScribeTask):
+    """Abort Import."""
+
+
+class SearchIndexSyncAbortTask(ScribeTask):
+    """Abort current search index sync."""

codex/librarian/scribe/timestamp_update.py

@@ -0,0 +1,135 @@
+"""Update Groups timestamp for cover cache busting."""
+
+from collections.abc import Mapping
+from datetime import datetime
+
+from django.db.models import QuerySet
+from django.db.models.aggregates import Count
+from django.db.models.functions.datetime import Now
+from django.db.models.query import Q
+from django.utils import timezone
+
+from codex.librarian.notifier.tasks import LIBRARY_CHANGED_TASK
+from codex.librarian.scribe.status import UpdateGroupTimestampsStatus
+from codex.librarian.worker import WorkerStatusMixin
+from codex.models import StoryArc, Volume
+from codex.models.groups import BrowserGroupModel
+from codex.models.library import Library
+from codex.views.const import GROUP_MODELS
+
+_UPDATE_FIELDS = ("updated_at",)
+
+
+class TimestampUpdater(WorkerStatusMixin):
+    """Update Groups timestamp for cover cache busting."""
+
+    @staticmethod
+    def _get_update_filter(
+        model: type[BrowserGroupModel],
+        start_time: datetime,
+        force_update_group_map: Mapping,
+        library: Library,
+    ):
+        # Get groups with comics updated during this import
+        rel = "storyarcnumber__" if model == StoryArc else ""
+        updated_at_rel = rel + "comic__updated_at__gt"
+        library_rel = rel + "comic__library"
+        updated_filter = {library_rel: library, updated_at_rel: start_time}
+        update_filter = Q(**updated_filter)
+
+        # Get groups with custom covers updated during this import
+        if model != Volume:
+            update_filter |= Q(custom_cover__updated_at__gt=start_time)
+
+        # Get groups to be force updated (usually those with deleted children)
+        if pks := force_update_group_map.get(model):
+            update_filter |= Q(pk__in=pks)
+
+        return update_filter
+
+    @staticmethod
+    def _add_child_count_filter(qs: QuerySet, model: type[BrowserGroupModel]):
+        """Filter out groups with no comics."""
+        rel_prefix = "storyarcnumber__" if model == StoryArc else ""
+        rel_prefix += "comic"
+        qs = qs.alias(child_count=Count(f"{rel_prefix}__pk", distinct=True))
+        return qs.filter(child_count__gt=0)
+
+    @classmethod
+    def _update_group_model(
+        cls,
+        force_update_group_map: Mapping,
+        model: type[BrowserGroupModel],
+        start_time: datetime,
+        library: Library,
+        log_list,
+    ):
+        """Update a single group model."""
+        update_filter = cls._get_update_filter(
+            model, start_time, force_update_group_map, library
+        )
+        qs = model.objects.filter(update_filter)
+        qs = cls._add_child_count_filter(qs, model)
+
+        qs = qs.distinct()
+        qs = qs.only(*_UPDATE_FIELDS)
+
+        updated = []
+        for obj in qs:
+            obj.updated_at = Now()
+            updated.append(obj)
+
+        count = len(updated)
+        if count:
+            model.objects.bulk_update(updated, _UPDATE_FIELDS)
+            log_list.append(f"{count} {model.__name__}s")
+        return count
+
+    def update_library_groups(
+        self,
+        library: Library,
+        start_time: datetime,
+        force_update_group_map: Mapping,
+        *,
+        mark_library_in_progress=False,
+    ):
+        """Update timestamps for each group for cover cache busting."""
+        total_count = 0
+        if mark_library_in_progress:
+            library.start_update()
+        status = UpdateGroupTimestampsStatus()
+        self.status_controller.start(status)
+        try:
+            log_list = []
+            for model in GROUP_MODELS:
+                count = self._update_group_model(
+                    force_update_group_map, model, start_time, library, log_list
+                )
+                if not count:
+                    continue
+                self.log.debug(f"Updated {count} {model.__name__}s timestamps.")
+                status.increment_complete(count)
+                self.status_controller.update(status)
+                total_count += count
+        finally:
+            if mark_library_in_progress:
+                library.end_update()
+            self.status_controller.finish(status)
+        return total_count
+
+    def update_groups(self, task):
+        """Update groups in all libraries."""
+        count = 0
+        start_time = task.start_time if task.start_time else timezone.now()
+        libraries = Library.objects.filter(covers_only=False).only("pk")
+        for library in libraries:
+            count += self.update_library_groups(
+                library, start_time, {}, mark_library_in_progress=True
+            )
+        level = "INFO" if count else "DEBUG"
+        self.log.log(level, f"Updated timestamps for {count} groups.")
+        self.librarian_queue.put(LIBRARY_CHANGED_TASK)
+
+    def __init__(self, log, librarian_queue, db_write_lock):
+        """Initialize Worker."""
+        self.init_worker(log, librarian_queue, db_write_lock)

codex/librarian/status.py

@@ -0,0 +1,76 @@
+"""Librarian Status dataclass."""
+
+from abc import ABC
+from dataclasses import dataclass
+from time import time
+from typing import ClassVar
+
+from humanize import intword, naturaldelta
+
+
+@dataclass
+class Status(ABC):
+    """Keep track of librarians status in memory."""
+
+    CODE: ClassVar[str]
+    VERB: ClassVar[str]
+    ITEM_NAME: ClassVar[str]
+    SINGLE: ClassVar[bool] = False
+    _title: ClassVar[str] = ""
+    _verbed: ClassVar[str] = ""
+
+    complete: int | None = None
+    total: int | None = None
+    since_updated: float = 0.0
+    subtitle: str = ""
+    start_time: float | None = None
+    log_success: bool = False
+
+    @classmethod
+    def title(cls):
+        """Return createed title."""
+        if not cls._title:
+            title_parts = (cls.VERB, *cls.ITEM_NAME.split(" "))
+            title_parts = (part.capitalize() for part in title_parts)
+            cls._title = " ".join(title_parts)
+        return cls._title
+
+    @classmethod
+    def verbed(cls):
+        """Return verbed, create it if it doesn't exist."""
+        if not cls._verbed:
+            cls._verbed = cls.VERB + "d"
+        return cls._verbed
+
+    def increment_complete(self, count: int = 1):
+        """Add count to complete."""
+        self.complete = self.complete + count if self.complete else count
+
+    def decrement_total(self):
+        """Decrement total if not not."""
+        self.total = max(self.total - 1, 0) if self.total is not None else None
+
+    def start(self):
+        """Set start time."""
+        self.start_time = time()
+
+    def _elapsed(self):
+        return time() - self.start_time if self.start_time else 0
+
+    def elapsed(self):
+        """Elapsed time."""
+        return naturaldelta(self._elapsed())
+
+    def per_second(self):
+        """Items per second."""
+        if self.SINGLE or self.total is None:
+            return ""
+        elapsed = self._elapsed()
+        ips = intword(self.total / elapsed) if elapsed else "infinite"
+        return f"{ips} {self.ITEM_NAME} per second"
+
+    def reset(self):
+        """Reset for batch statii."""
+        self.complete = 0
+        self.total = 0
+        self.start()