codex 1.6.0a4__py3-none-any.whl → 1.6.0a6__py3-none-any.whl

codex/librarian/importer/create_fks.py

@@ -3,13 +3,17 @@
 So we may safely create the comics next.
 """
 
-from itertools import chain
 from pathlib import Path
 
+from django.core.exceptions import ObjectDoesNotExist
+from django.db.models.functions.datetime import Now
+
 from codex.librarian.importer.const import (
     BULK_UPDATE_FOLDER_FIELDS,
+    CLASS_CUSTOM_COVER_GROUP_MAP,
     COUNT_FIELDS,
     CREATE_DICT_UPDATE_FIELDS,
+    CUSTOM_COVER_UPDATE_FIELDS,
     GROUP_BASE_FIELDS,
     GROUP_UPDATE_FIELDS,
     IMPRINT,
@@ -24,6 +28,7 @@ from codex.models import (
     Contributor,
     ContributorPerson,
     ContributorRole,
+    CustomCover,
     Folder,
     Imprint,
     Publisher,
@@ -41,7 +46,20 @@ class CreateForeignKeysMixin(QueuedThread):
     """Methods for creating foreign keys."""
 
     @staticmethod
-    def _create_group_obj(group_class, group_param_tuple, group_count):
+    def _add_custom_cover_to_group(group_class, obj):
+        """If a custom cover exists for this group, add it."""
+        group = CLASS_CUSTOM_COVER_GROUP_MAP.get(group_class)
+        if not group:
+            # Normal, volume doesn't link to covers
+            return
+        try:
+            cover = CustomCover.objects.filter(group=group, sort_name=obj.sort_name)[0]
+            obj.custom_cover = cover
+        except (IndexError, ObjectDoesNotExist):
+            pass
+
+    @classmethod
+    def _create_group_obj(cls, group_class, group_param_tuple, group_count):
         """Create a set of browser group objects."""
         defaults = {"name": group_param_tuple[-1]}
         if group_class in (Imprint, Series, Volume):
@@ -64,6 +82,7 @@ class CreateForeignKeysMixin(QueuedThread):
 
         obj = group_class(**defaults)
         obj.presave()
+        cls._add_custom_cover_to_group(group_class, obj)
         return obj
 
     @staticmethod
@@ -99,6 +118,8 @@ class CreateForeignKeysMixin(QueuedThread):
             obj = self._create_group_obj(group_class, group_param_tuple, group_count)
             create_groups.append(obj)
         update_fields = GROUP_UPDATE_FIELDS[group_class]
+        if group_class in CLASS_CUSTOM_COVER_GROUP_MAP:
+            update_fields += ("custom_cover",)
         group_class.objects.bulk_create(
             create_groups,
             update_conflicts=True,
@@ -157,6 +178,7 @@ class CreateForeignKeysMixin(QueuedThread):
             parent_folder=parent,
         )
         folder.presave()
+        self._add_custom_cover_to_group(Folder, folder)
         create_folders.append(folder)
 
     def _bulk_folders_create_depth_level(self, library, paths, status):
@@ -212,6 +234,7 @@ class CreateForeignKeysMixin(QueuedThread):
             named_obj = named_class(name=name)
             if is_story_arc:
                 named_obj.presave()
+                self._add_custom_cover_to_group(StoryArc, named_obj)
             create_named_objs.append(named_obj)
 
         update_fields = NAMED_MODEL_UPDATE_FIELDS
@@ -282,8 +305,8 @@ class CreateForeignKeysMixin(QueuedThread):
             self.status_controller.update(status)
         return count
 
-    @staticmethod
-    def _get_create_fks_totals(create_data):
+    def create_all_fks(self, library, create_data) -> int:
+        """Bulk create all foreign keys."""
         (
             create_groups,
             update_groups,
@@ -292,35 +315,12 @@ class CreateForeignKeysMixin(QueuedThread):
             create_contributors,
             create_story_arc_numbers,
             create_identifiers,
+            total_fks,
         ) = create_data
-        total_fks = 0
-        for data_group in chain(
-            create_groups.values(), update_groups.values(), create_fks.values()
-        ):
-            total_fks += len(data_group)
-        total_fks += (
-            len(create_folder_paths)
-            + len(create_contributors)
-            + len(create_story_arc_numbers)
-            + len(create_identifiers)
-        )
-        return total_fks
 
-    def create_all_fks(self, library, create_data) -> int:
-        """Bulk create all foreign keys."""
-        total_fks = self._get_create_fks_totals(create_data)
         status = Status(ImportStatusTypes.CREATE_FKS, 0, total_fks)
         try:
             self.status_controller.start(status)
-            (
-                create_groups,
-                update_groups,
-                create_folder_paths,
-                create_fks,
-                create_contributors,
-                create_story_arc_numbers,
-                create_identifiers,
-            ) = create_data
 
             for group_class, group_tree_counts in create_groups.items():
                 count = self._bulk_group_create(
@@ -375,3 +375,63 @@ class CreateForeignKeysMixin(QueuedThread):
         finally:
             self.status_controller.finish(status)
         return status.complete if status.complete else 0
+
+    @status_notify(ImportStatusTypes.COVERS_MODIFIED, updates=False)
+    def update_custom_covers(
+        self, update_covers_qs, link_cover_pks, status=None
+    ) -> int:
+        """Update Custom Covers."""
+        count = 0
+        update_covers_count = update_covers_qs.count()
+        if not update_covers_count:
+            return count
+        if status:
+            status.total = update_covers_count
+        now = Now()
+
+        update_covers = []
+        for cover in update_covers_qs.only(*CUSTOM_COVER_UPDATE_FIELDS):
+            cover.updated_at = now
+            cover.presave()
+            update_covers.append(cover)
+
+        if update_covers:
+            CustomCover.objects.bulk_update(update_covers, CUSTOM_COVER_UPDATE_FIELDS)
+            update_cover_pks = update_covers_qs.values_list("pk", flat=True)
+            link_cover_pks.update(update_cover_pks)
+            self._remove_covers(update_cover_pks, custom=True)  # type: ignore
+            count = len(update_covers)
+            if status:
+                status.add_complete(count)
+        return count
+
+    @status_notify(ImportStatusTypes.COVERS_CREATED, updates=False)
+    def create_custom_covers(
+        self, create_cover_paths, library, link_cover_pks, status=None
+    ) -> int:
+        """Create Custom Covers."""
+        count = 0
+        if not create_cover_paths:
+            return count
+        if status:
+            status.total = len(create_cover_paths)
+
+        create_covers = []
+        for path in create_cover_paths:
+            cover = CustomCover(library=library, path=path)
+            cover.presave()
+            create_covers.append(cover)
+
+        if create_covers:
+            objs = CustomCover.objects.bulk_create(
+                create_covers,
+                update_conflicts=True,
+                update_fields=("path", "stat"),
+                unique_fields=CustomCover._meta.unique_together[0],
+            )
+            created_pks = frozenset(obj.pk for obj in objs)
+            link_cover_pks.update(created_pks)
+            count = len(created_pks)
+            if status:
+                status.add_complete(count)
+        return count

codex/librarian/importer/deleted.py

@@ -1,16 +1,19 @@
 """Clean up the database after moves or imports."""
 
 from codex.librarian.covers.tasks import CoverRemoveTask
+from codex.librarian.importer.const import COMIC_GROUP_FIELD_NAMES
 from codex.librarian.importer.status import ImportStatusTypes, status_notify
-from codex.models import Comic, Folder
+from codex.models import Comic, Folder, StoryArc
+from codex.models.paths import CustomCover
+from codex.settings.settings import MAX_CHUNK_SIZE
 from codex.threads import QueuedThread
 
 
 class DeletedMixin(QueuedThread):
     """Clean up database methods."""
 
-    def _remove_covers(self, delete_comic_pks):
-        task = CoverRemoveTask(delete_comic_pks)
+    def _remove_covers(self, delete_pks, custom=False):
+        task = CoverRemoveTask(delete_pks, custom)
         self.librarian_queue.put(task)
 
     @status_notify(status_type=ImportStatusTypes.DIRS_DELETED, updates=False)
@@ -36,14 +39,47 @@ class DeletedMixin(QueuedThread):
             )
         return count
 
+    @staticmethod
+    def _populate_deleted_comic_groups(delete_qs, deleted_comic_groups):
+        """Populate changed groups for cover timestamp updater."""
+        comics_deleted_qs = delete_qs.only(*COMIC_GROUP_FIELD_NAMES).prefetch_related(
+            "story_arc_numbers__story_arc"
+        )
+        for comic in comics_deleted_qs.iterator(chunk_size=MAX_CHUNK_SIZE):
+            for field_name in COMIC_GROUP_FIELD_NAMES:
+                if field_name == "story_arc_numbers":
+                    for san in comic.story_arc_numbers.select_related("story_arc").only(
+                        "story_arc"
+                    ):
+                        deleted_comic_groups[StoryArc].add(san.story_arc.pk)
+                elif field_name == "folders":
+                    for folder in comic.folders.only("pk"):
+                        deleted_comic_groups[Folder].add(folder.pk)
+                else:
+                    related_model = comic._meta.get_field(field_name).related_model
+                    related_id = getattr(comic, field_name).pk
+                    deleted_comic_groups[related_model].add(related_id)
+
+    @staticmethod
+    def _init_deleted_comic_groups(deleted_comic_groups):
+        """Init deleted_comic_groups, used later even if no deletes."""
+        for field_name in COMIC_GROUP_FIELD_NAMES:
+            related_model = Comic._meta.get_field(field_name).related_model
+            deleted_comic_groups[related_model] = set()
+
     @status_notify(status_type=ImportStatusTypes.FILES_DELETED, updates=False)
-    def _bulk_comics_deleted(self, delete_comic_paths, library, **kwargs):
+    def _bulk_comics_deleted(
+        self, delete_comic_paths, library, deleted_comic_groups, **kwargs
+    ):
         """Bulk delete comics found missing from the filesystem."""
         if not delete_comic_paths:
             return 0
-        comics = Comic.objects.filter(library=library, path__in=delete_comic_paths)
-        delete_comic_pks = frozenset(comics.values_list("pk", flat=True))
-        comics.delete()
+        delete_qs = Comic.objects.filter(library=library, path__in=delete_comic_paths)
+
+        self._populate_deleted_comic_groups(delete_qs, deleted_comic_groups)
+
+        delete_comic_pks = frozenset(delete_qs.values_list("pk", flat=True))
+        delete_qs.delete()
 
         self._remove_covers(delete_comic_pks)
 
@@ -53,11 +89,37 @@ class DeletedMixin(QueuedThread):
 
         return count
 
-    def delete(self, library, task):
+    @status_notify(status_type=ImportStatusTypes.COVERS_DELETED, updates=False)
+    def _bulk_covers_deleted(self, delete_cover_paths, library, **kwargs):
+        """Bulk delete comics found missing from the filesystem."""
+        if not delete_cover_paths:
+            return 0
+        covers = CustomCover.objects.filter(
+            library=library, path__in=delete_cover_paths
+        )
+        delete_cover_pks = frozenset(covers.values_list("pk", flat=True))
+        covers.delete()
+
+        self._remove_covers(delete_cover_pks, custom=True)
+
+        count = len(delete_cover_paths)
+        if count:
+            self.log.info(f"Deleted {count} custom covers from {library.path}")
+
+        return count
+
+    def delete(self, library, task, deleted_comic_groups):
         """Delete files and folders."""
         count = self._bulk_folders_deleted(task.dirs_deleted, library)
         task.dirs_deleted = None
 
-        count += self._bulk_comics_deleted(task.files_deleted, library)
+        self._init_deleted_comic_groups(deleted_comic_groups)
+        count += self._bulk_comics_deleted(
+            task.files_deleted, library, deleted_comic_groups
+        )
         task.files_deleted = None
+
+        count += self._bulk_covers_deleted(task.covers_deleted, library)
+        task.covers_deleted = None
+
         return count

codex/librarian/importer/importerd.py

@@ -5,10 +5,12 @@ from pathlib import Path
 from time import sleep, time
 
 from django.core.cache import cache
+from django.utils.timezone import now
 from humanize import naturaldelta
 
 from codex.librarian.importer.aggregate_metadata import AggregateMetadataMixin
 from codex.librarian.importer.const import FIS, FKS, M2M_MDS, MDS
+from codex.librarian.importer.covers import CoversMixin
 from codex.librarian.importer.deleted import DeletedMixin
 from codex.librarian.importer.failed_imports import FailedImportsMixin
 from codex.librarian.importer.moved import MovedMixin
@@ -35,6 +37,7 @@ class ComicImporterThread(
     UpdateComicsMixin,
     FailedImportsMixin,
     MovedMixin,
+    CoversMixin,
 ):
     """A worker to handle all bulk database updates."""
 
@@ -117,6 +120,29 @@ class ComicImporterThread(
             log += ", ".join(dirs_log)
             self.log.debug("  " + log)
 
+    @staticmethod
+    def _init_librarian_status_moved(task, status_list):
+        """Initialize moved statuses."""
+        search_index_updates = 0
+        if task.dirs_moved:
+            status_list += [
+                Status(ImportStatusTypes.DIRS_MOVED, None, len(task.dirs_moved))
+            ]
+        if task.files_moved:
+            status_list += [
+                Status(ImportStatusTypes.FILES_MOVED, None, len(task.files_moved))
+            ]
+            search_index_updates += len(task.files_moved)
+        if task.covers_moved:
+            status_list += [
+                Status(ImportStatusTypes.COVERS_MOVED, None, len(task.covers_moved))
+            ]
+        if task.dirs_modified:
+            status_list += [
+                Status(ImportStatusTypes.DIRS_MODIFIED, None, len(task.dirs_modified))
+            ]
+        return search_index_updates
+
     @staticmethod
     def _init_if_modified_or_created(task, path, status_list):
         """Initialize librarian statuses for modified or created ops."""
@@ -124,6 +150,7 @@ class ComicImporterThread(
         status_list += [
             Status(ImportStatusTypes.AGGREGATE_TAGS, 0, total_paths, subtitle=path),
             Status(ImportStatusTypes.QUERY_MISSING_FKS),
+            Status(ImportStatusTypes.QUERY_MISSING_COVERS),
             Status(ImportStatusTypes.CREATE_FKS),
         ]
         if task.files_modified:
@@ -134,40 +161,56 @@ class ComicImporterThread(
                     len(task.files_modified),
                 )
             ]
-        if task.files_created:
+        if task.covers_modified:
             status_list += [
-                Status(ImportStatusTypes.FILES_CREATED, None, len(task.files_created))
+                Status(
+                    ImportStatusTypes.COVERS_MODIFIED,
+                    None,
+                    len(task.covers_modified),
+                )
             ]
-        if task.files_modified or task.files_created:
-            status_list += [Status(ImportStatusTypes.LINK_M2M_FIELDS)]
-        return total_paths
 
-    def _init_librarian_status(self, task, path):
-        """Update the librarian status tasks."""
-        status_list = []
-        search_index_updates = 0
-        if task.dirs_moved:
+        if task.files_created or task.covers_created:
             status_list += [
-                Status(ImportStatusTypes.DIRS_MOVED, None, len(task.dirs_moved))
+                Status(ImportStatusTypes.FILES_CREATED, None, len(task.files_created))
             ]
-        if task.files_moved:
+        if task.covers_created:
             status_list += [
-                Status(ImportStatusTypes.FILES_MOVED, None, len(task.files_moved))
+                Status(ImportStatusTypes.COVERS_CREATED, None, len(task.covers_created))
             ]
-            search_index_updates += len(task.files_moved)
-        if task.files_modified:
+
+        if task.files_modified or task.files_created:
+            status_list += [Status(ImportStatusTypes.LINK_M2M_FIELDS)]
+
+        num_covers_linked = (
+            len(task.covers_moved)
+            + len(task.covers_modified)
+            + len(task.covers_created)
+        )
+        if num_covers_linked:
             status_list += [
-                Status(ImportStatusTypes.DIRS_MODIFIED, None, len(task.dirs_modified))
+                Status(ImportStatusTypes.COVERS_LINK, None, num_covers_linked)
             ]
-        if task.files_modified or task.files_created:
-            search_index_updates += self._init_if_modified_or_created(
-                task, path, status_list
-            )
+        return total_paths
+
+    @staticmethod
+    def _init_librarian_status_deleted(task, status_list):
+        """Init deleted statuses."""
+        search_index_updates = 0
         if task.files_deleted:
             status_list += [
                 Status(ImportStatusTypes.FILES_DELETED, None, len(task.files_deleted))
             ]
             search_index_updates += len(task.files_deleted)
+        if task.covers_deleted:
+            status_list += [
+                Status(ImportStatusTypes.COVERS_DELETED, None, len(task.covers_deleted))
+            ]
+        return search_index_updates
+
+    @staticmethod
+    def _init_librarian_status_search_index(search_index_updates, status_list):
+        """Init search index statuses."""
         status_list += [
             Status(
                 SearchIndexStatusTypes.SEARCH_INDEX_UPDATE,
@@ -176,6 +219,23 @@ class ComicImporterThread(
             ),
             Status(SearchIndexStatusTypes.SEARCH_INDEX_REMOVE),
         ]
+
+    def _init_librarian_status(self, task, path):
+        """Update the librarian status tasks."""
+        status_list = []
+        search_index_updates = 0
+        search_index_updates += self._init_librarian_status_moved(task, status_list)
+        if (
+            task.files_modified
+            or task.files_created
+            or task.covers_modified
+            or task.covers_created
+        ):
+            search_index_updates += self._init_if_modified_or_created(
+                task, path, status_list
+            )
+        search_index_updates += self._init_librarian_status_deleted(task, status_list)
+        self._init_librarian_status_search_index(search_index_updates, status_list)
         self.status_controller.start_many(status_list)
 
     def _init_apply(self, library, task):
@@ -193,12 +253,38 @@ class ComicImporterThread(
         self._log_task(library.path, task)
         self._init_librarian_status(task, library.path)
 
-    def _create_comic_relations(self, library, fks) -> int:
+    def _create_comic_and_cover_relations(
+        self, library, fks, cover_paths: frozenset[int], link_cover_pks: set[int]
+    ) -> int:
         """Query all foreign keys to determine what needs creating, then create them."""
-        if not fks:
-            return 0
+        count = 0
+        if not fks and not cover_paths:
+            return count
         create_data = self.query_all_missing_fks(library.path, fks)
-        return self.create_all_fks(library, create_data)
+        query_cover_data = []
+        count += self.query_missing_custom_covers(
+            cover_paths,
+            library,
+            query_cover_data,
+        )
+        count += self.create_all_fks(library, create_data)
+        if query_cover_data:
+            update_covers_qs, create_cover_paths = query_cover_data
+            count += self.update_custom_covers(update_covers_qs, link_cover_pks)
+            link_covers_status = Status(
+                ImportStatusTypes.COVERS_LINK, 0, len(link_cover_pks)
+            )
+            self.status_controller.update(link_covers_status, notify=False)
+            count += self.create_custom_covers(
+                create_cover_paths,
+                library,
+                link_cover_pks,
+            )
+            link_covers_status = Status(
+                ImportStatusTypes.COVERS_LINK, 0, len(link_cover_pks)
+            )
+            self.status_controller.update(link_covers_status, notify=False)
+        return count
 
     def _finish_apply_status(self, library):
         """Finish all librarian statuses."""
@@ -239,13 +325,16 @@ class ComicImporterThread(
         library = Library.objects.get(pk=task.library_id)
         try:
             self._init_apply(library, task)
+            import_start_time = now()
 
             changed: int = 0
-            changed += self.move_and_modify_dirs(library, task)
+            link_cover_pks: set[int] = set()
+            changed += self.move_and_modify_dirs(library, task, link_cover_pks)
 
             modified_paths = task.files_modified
             created_paths = task.files_created
             task.files_modified = task.files_created = None
+
             mds = {}
             m2m_mds = {}
             fks = {}
@@ -266,8 +355,13 @@ class ComicImporterThread(
             modified_paths -= fis.keys()
             created_paths -= fis.keys()
 
-            changed += self._create_comic_relations(library, fks)
-            fks = None
+            cover_paths = frozenset(task.covers_modified | task.covers_created)
+            task.covers_modified = task.covers_created = None
+
+            changed += self._create_comic_and_cover_relations(
+                library, fks, cover_paths, link_cover_pks
+            )
+            fks = cover_paths = None
 
             imported_count = self.bulk_update_comics(
                 modified_paths,
@@ -282,11 +376,18 @@ class ComicImporterThread(
             m2m_mds = None
             changed += imported_count
 
+            self.link_custom_covers(link_cover_pks)
+            link_cover_pks = set()
+
             new_failed_imports = self.fail_imports(
                 library, fis, bool(task.files_deleted)
             )
 
-            changed += self.delete(library, task)
+            deleted_comic_groups = {}
+            changed += self.delete(library, task, deleted_comic_groups)
+
+            self.update_groups_for_covers(import_start_time, deleted_comic_groups)
+
             if changed:
                 cache.clear()
         finally:
@@ -310,11 +411,15 @@ class ComicImporterThread(
                 library_id=library_id,
                 dirs_moved={},
                 files_moved={},
+                covers_moved={},
                 dirs_modified=frozenset(),
                 files_modified=frozenset(paths),
+                covers_modified=frozenset(),
                 files_created=frozenset(),
+                covers_created=frozenset(),
                 dirs_deleted=frozenset(),
                 files_deleted=frozenset(),
+                covers_deleted=frozenset(),
                 force_import_metadata=True,
             )
         self._apply(task)

codex/librarian/importer/link_comics.py

@@ -5,6 +5,7 @@ from pathlib import Path
 from django.db.models import Q
 
 from codex.librarian.importer.const import (
+    CLASS_CUSTOM_COVER_GROUP_MAP,
     COMIC_FK_FIELD_NAMES,
     DICT_MODEL_FIELD_NAME_CLASS_MAP,
     DICT_MODEL_REL_LINK_MAP,
@@ -20,6 +21,7 @@ from codex.librarian.importer.const import (
 from codex.librarian.importer.status import ImportStatusTypes, status_notify
 from codex.models import (
     Comic,
+    CustomCover,
     Folder,
     Imprint,
     Publisher,
@@ -257,3 +259,56 @@ class LinkComicsMixin(QueuedThread):
         if del_total:
             self.log.info(f"Deleted {del_total} stale relations for altered comics.")
         return created_total + del_total
+
+    def _link_custom_cover_prepare(self, cover, model_map):
+        """Prepare one cover in the model map for bulk update."""
+        if cover.library and cover.library.covers_only:
+            model = CLASS_CUSTOM_COVER_GROUP_MAP.inverse.get(cover.group)
+            if not model:
+                self.log.warning(f"Custom Cover model not found for {cover.path}")
+                return
+            group_filter = {"sort_name": cover.sort_name}
+        else:
+            model = Folder
+            path = str(Path(cover.path).parent)
+            group_filter = {"path": path}
+        qs = model.objects.filter(**group_filter).exclude(custom_cover=cover)
+        if not qs.exists():
+            return
+
+        if model not in model_map:
+            model_map[model] = []
+
+        for obj in qs.iterator():
+            obj.custom_cover = cover
+            model_map[model].append(obj)
+
+    def _link_custom_cover_group(self, model, objs, status):
+        """Bulk link a group to it's custom covers."""
+        count = 0
+        if not objs:
+            return count
+        model.objects.bulk_update(objs, ["custom_cover"])
+        count += len(objs)
+        self.log.info(f"Linked {count} custom covers to {model.__name__}s")
+        if status:
+            status.complete = status.complete or 0
+            status.complete += count
+        return count
+
+    @status_notify(status_type=ImportStatusTypes.COVERS_LINK)
+    def link_custom_covers(self, link_cover_pks, status=None):
+        """Link Custom Covers to Groups."""
+        # Aggregate objs to update for each group model.
+        model_map = {}
+        covers = CustomCover.objects.filter(pk__in=link_cover_pks).only(
+            "library", "path"
+        )
+        for cover in covers:
+            self._link_custom_cover_prepare(cover, model_map)
+
+        # Bulk update each model type
+        total_count = 0
+        for model, objs in model_map.items():
+            total_count += self._link_custom_cover_group(model, objs, status)
+        return total_count