codex 1.5.19__py3-none-any.whl → 1.6.0__py3-none-any.whl

codex/librarian/importer/create_covers.py

@@ -0,0 +1,96 @@
+"""Create Custom Covers."""
+
+from django.db.models.functions.datetime import Now
+
+from codex.librarian.importer.const import (
+    COVERS_CREATE,
+    COVERS_UPDATE,
+    CUSTOM_COVER_UPDATE_FIELDS,
+    LINK_COVER_PKS,
+)
+from codex.librarian.importer.create_comics import CreateComicsImporter
+from codex.librarian.importer.status import ImportStatusTypes
+from codex.models import (
+    CustomCover,
+)
+from codex.status import Status
+
+
+class CreateCoversImporter(CreateComicsImporter):
+    """Create Custom Covers."""
+
+    def update_custom_covers(self, status=None):
+        """Update Custom Covers."""
+        update_covers_qs = self.metadata.pop(COVERS_UPDATE, None)
+        if not update_covers_qs:
+            return
+        update_covers_count = update_covers_qs.count()
+        if not update_covers_count:
+            return
+        status = Status(ImportStatusTypes.COVERS_MODIFIED, 0, update_covers_count)
+        self.status_controller.start(status)
+
+        update_covers = []
+        for cover in update_covers_qs.only(*CUSTOM_COVER_UPDATE_FIELDS):
+            cover.updated_at = Now()
+            cover.presave()
+            update_covers.append(cover)
+
+        count = 0
+        if update_covers:
+            CustomCover.objects.bulk_update(update_covers, CUSTOM_COVER_UPDATE_FIELDS)
+            update_cover_pks = update_covers_qs.values_list("pk", flat=True)
+            if LINK_COVER_PKS not in self.metadata:
+                self.metadata[LINK_COVER_PKS] = set()
+            self.metadata[LINK_COVER_PKS].update(update_cover_pks)
+            self._remove_covers(update_cover_pks, custom=True)  # type: ignore
+            count = len(update_covers)
+            if status:
+                status.add_complete(count)
+
+            link_covers_status = Status(
+                ImportStatusTypes.COVERS_LINK, 0, len(self.metadata[LINK_COVER_PKS])
+            )
+            self.status_controller.update(link_covers_status, notify=False)
+
+        self.changed += count
+        self.status_controller.finish(status)
+
+    def create_custom_covers(self, status=None):
+        """Create Custom Covers."""
+        create_cover_paths = self.metadata.pop(COVERS_CREATE, ())
+        num_create_cover_paths = len(create_cover_paths)
+        if not num_create_cover_paths:
+            return
+        status = Status(ImportStatusTypes.COVERS_MODIFIED, 0, num_create_cover_paths)
+        self.status_controller.start(status)
+
+        create_covers = []
+        for path in create_cover_paths:
+            cover = CustomCover(library=self.library, path=path)
+            cover.presave()
+            create_covers.append(cover)
+
+        count = 0
+        if create_covers:
+            objs = CustomCover.objects.bulk_create(
+                create_covers,
+                update_conflicts=True,
+                update_fields=("path", "stat"),
+                unique_fields=CustomCover._meta.unique_together[0],
+            )
+            created_pks = frozenset(obj.pk for obj in objs)
+            if LINK_COVER_PKS not in self.metadata:
+                self.metadata[LINK_COVER_PKS] = set()
+            self.metadata[LINK_COVER_PKS].update(created_pks)
+            count = len(created_pks)
+            if status:
+                status.add_complete(count)
+
+            link_covers_status = Status(
+                ImportStatusTypes.COVERS_LINK, 0, len(self.metadata[LINK_COVER_PKS])
+            )
+            self.status_controller.update(link_covers_status, notify=False)
+
+        self.changed += count
+        self.status_controller.finish(status)

codex/librarian/importer/create_fks.py

@@ -3,13 +3,25 @@
 So we may safely create the comics next.
 """
 
-from itertools import chain
 from pathlib import Path
 
+from django.core.exceptions import ObjectDoesNotExist
+
 from codex.librarian.importer.const import (
-    BULK_UPDATE_FOLDER_MODIFIED_FIELDS,
+    BULK_UPDATE_FOLDER_FIELDS,
+    CLASS_CUSTOM_COVER_GROUP_MAP,
     COUNT_FIELDS,
     CREATE_DICT_UPDATE_FIELDS,
+    FK_CREATE,
+    FKC_CONTRIBUTORS,
+    FKC_CREATE_FKS,
+    FKC_CREATE_GROUPS,
+    FKC_FOLDER_PATHS,
+    FKC_IDENTIFIERS,
+    FKC_STORY_ARC_NUMBERS,
+    FKC_TOTAL_FKS,
+    FKC_UPDATE_GROUPS,
+    GROUP_BASE_FIELDS,
     GROUP_UPDATE_FIELDS,
     IMPRINT,
     ISSUE_COUNT,
@@ -18,11 +30,13 @@ from codex.librarian.importer.const import (
     SERIES,
     VOLUME_COUNT,
 )
-from codex.librarian.importer.status import ImportStatusTypes, status_notify
+from codex.librarian.importer.create_covers import CreateCoversImporter
+from codex.librarian.importer.status import ImportStatusTypes
 from codex.models import (
     Contributor,
     ContributorPerson,
     ContributorRole,
+    CustomCover,
     Folder,
     Imprint,
     Publisher,
@@ -33,14 +47,26 @@ from codex.models import (
 )
 from codex.models.named import Identifier, IdentifierType
 from codex.status import Status
-from codex.threads import QueuedThread
 
 
-class CreateForeignKeysMixin(QueuedThread):
+class CreateForeignKeysImporter(CreateCoversImporter):
     """Methods for creating foreign keys."""
 
     @staticmethod
-    def _create_group_obj(group_class, group_param_tuple, group_count):
+    def _add_custom_cover_to_group(group_class, obj):
+        """If a custom cover exists for this group, add it."""
+        group = CLASS_CUSTOM_COVER_GROUP_MAP.get(group_class)
+        if not group:
+            # Normal, volume doesn't link to covers
+            return
+        try:
+            cover = CustomCover.objects.filter(group=group, sort_name=obj.sort_name)[0]
+            obj.custom_cover = cover
+        except (IndexError, ObjectDoesNotExist):
+            pass
+
+    @classmethod
+    def _create_group_obj(cls, group_class, group_param_tuple, group_count):
         """Create a set of browser group objects."""
         defaults = {"name": group_param_tuple[-1]}
         if group_class in (Imprint, Series, Volume):
@@ -61,7 +87,10 @@ class CreateForeignKeysMixin(QueuedThread):
             )
             defaults[ISSUE_COUNT] = group_count
 
-        return group_class(**defaults)
+        obj = group_class(**defaults)
+        obj.presave()
+        cls._add_custom_cover_to_group(group_class, obj)
+        return obj
 
     @staticmethod
     def _update_group_obj(group_class, group_param_tuple, group_count):
@@ -85,17 +114,18 @@ class CreateForeignKeysMixin(QueuedThread):
             obj = None
         return obj
 
-    @status_notify()
-    def _bulk_group_create(self, group_tree_counts, group_class, status=None) -> int:
+    def _bulk_group_create(self, group_tree_counts, group_class, status):
         """Bulk creates groups."""
         count = 0
         if not group_tree_counts:
-            return count
+            return
         create_groups = []
         for group_param_tuple, group_count in group_tree_counts.items():
             obj = self._create_group_obj(group_class, group_param_tuple, group_count)
             create_groups.append(obj)
         update_fields = GROUP_UPDATE_FIELDS[group_class]
+        if group_class in CLASS_CUSTOM_COVER_GROUP_MAP:
+            update_fields += ("custom_cover",)
         group_class.objects.bulk_create(
             create_groups,
             update_conflicts=True,
@@ -106,18 +136,14 @@ class CreateForeignKeysMixin(QueuedThread):
         if count:
             vnp = group_class._meta.verbose_name_plural.title()
             self.log.info(f"Created {count} {vnp}.")
-        if status:
-            status.complete = status.complete or 0
-            status.complete += count
-            self.status_controller.update(status)
-        return count
+        status.add_complete(count)
+        self.status_controller.update(status)
+        return
 
-    @status_notify()
-    def _bulk_group_updater(self, group_tree_counts, group_class, status=None):
+    def _bulk_group_updater(self, group_tree_counts, group_class, status):
         """Bulk update groups."""
-        count = 0
         if not group_tree_counts:
-            return count
+            return
         update_groups = []
         for group_param_tuple, group_count in group_tree_counts.items():
             obj = self._update_group_obj(group_class, group_param_tuple, group_count)
@@ -125,61 +151,56 @@ class CreateForeignKeysMixin(QueuedThread):
                 update_groups.append(obj)
         count_field = COUNT_FIELDS[group_class]
         group_class.objects.bulk_update(update_groups, fields=[count_field])
-        count += len(update_groups)
+        count = len(update_groups)
         if count:
             vnp = group_class._meta.verbose_name_plural.title()
             self.log.info(f"Updated {count} {vnp}.")
-        if status:
-            status.complete = status.complete or 0
-            status.complete += count
-            self.status_controller.update(status)
-        return count
+        status.add_complete(count)
+        self.status_controller.update(status)
 
-    def _bulk_folders_create_add_folder(self, library, path, create_folders):
+    def _bulk_folders_create_add_folder(self, path, create_folders):
         """Add one folder to the create list."""
         parent_path = str(path.parent)
         parent = None
         try:
             parent = Folder.objects.get(path=parent_path)
         except Folder.DoesNotExist:
-            if path.parent != Path(library.path):
+            if path.parent != Path(self.library.path):
                 self.log.warning(
                     f"Can't find parent folder {parent_path}"
-                    f" for {path} in library {library.path}"
+                    f" for {path} in library {self.library.path}"
                 )
         folder = Folder(
-            library=library,
+            library=self.library,
             path=str(path),
             name=path.name,
             parent_folder=parent,
         )
-        folder.set_stat()
+        folder.presave()
+        self._add_custom_cover_to_group(Folder, folder)
         create_folders.append(folder)
 
-    def _bulk_folders_create_depth_level(self, library, paths, status):
+    def _bulk_folders_create_depth_level(self, paths, status):
         """Create a depth level of folders."""
         create_folders = []
         for path in sorted(paths):
-            self._bulk_folders_create_add_folder(library, path, create_folders)
+            self._bulk_folders_create_add_folder(path, create_folders)
         Folder.objects.bulk_create(
             create_folders,
             update_conflicts=True,
-            update_fields=BULK_UPDATE_FOLDER_MODIFIED_FIELDS,
+            update_fields=BULK_UPDATE_FOLDER_FIELDS,
             unique_fields=Folder._meta.unique_together[0],  # type: ignore
         )
         count = len(create_folders)
-        if status:
-            status.complete = status.complete or 0
-            status.complete += count
-            self.status_controller.update(status)
+        status.add_complete(count)
+        self.status_controller.update(status)
         return count
 
-    @status_notify()
-    def bulk_folders_create(self, folder_paths: set, library, status=None):
+    def bulk_folders_create(self, folder_paths: frozenset, status):
         """Create folders breadth first."""
         count = 0
         if not folder_paths:
-            return count
+            return
         # group folder paths by depth
         folder_path_dict = {}
         for path_str in folder_paths:
@@ -190,66 +211,82 @@ class CreateForeignKeysMixin(QueuedThread):
             folder_path_dict[path_length].add(path)
 
         # create each depth level first to ensure we can assign parents
-        for _, paths in sorted(folder_path_dict.items()):
-            count += self._bulk_folders_create_depth_level(library, paths, status)
+        for depth_level in sorted(folder_path_dict):
+            paths = sorted(folder_path_dict[depth_level])
+            count += self._bulk_folders_create_depth_level(paths, status)
+            self.log.debug(f"Created {count} Folders at depth level {depth_level}")
 
         if count:
             self.log.info(f"Created {count} Folders.")
-        return count
+        self.status_controller.update(status)
 
-    @status_notify()
-    def _bulk_create_named_models(self, names, named_class, status=None):
+    def _bulk_create_named_models(self, names, named_class, status):
         """Bulk create named models."""
         count = len(names)
         if not count:
-            return count
+            return
         create_named_objs = []
+        is_story_arc = named_class == StoryArc
         for name in names:
             named_obj = named_class(name=name)
+            if is_story_arc:
+                named_obj.presave()
+                self._add_custom_cover_to_group(StoryArc, named_obj)
             create_named_objs.append(named_obj)
 
+        update_fields = NAMED_MODEL_UPDATE_FIELDS
+        if is_story_arc:
+            update_fields += GROUP_BASE_FIELDS
+
         named_class.objects.bulk_create(
             create_named_objs,
             update_conflicts=True,
-            update_fields=NAMED_MODEL_UPDATE_FIELDS,
+            update_fields=update_fields,
             unique_fields=named_class._meta.unique_together[0],
         )
         if count:
             vnp = named_class._meta.verbose_name_plural.title()
             self.log.info(f"Created {count} {vnp}.")
-        if status:
-            status.complete = status.complete or 0
-            status.complete += count
-            self.status_controller.update(status)
-        return count
+        status.add_complete(count)
+        self.status_controller.update(status)
 
     @staticmethod
-    def _create_contributor(role_name, person_name):
-        role = ContributorRole.objects.get(name=role_name) if role_name else None
-        person = ContributorPerson.objects.get(name=person_name)
-        return Contributor(role=role, person=person)
+    def _create_contributor_args(values):
+        return {
+            "role": ContributorRole.objects.get(name=values[0]) if values[0] else None,
+            "person": ContributorPerson.objects.get(name=values[1]),
+        }
 
     @staticmethod
-    def _create_story_arc_number(name, number):
-        story_arc = StoryArc.objects.get(name=name)
-        return StoryArcNumber(story_arc=story_arc, number=number)
+    def _create_story_arc_number_args(values):
+        return {"story_arc": StoryArc.objects.get(name=values[0]), "number": values[1]}
 
     @staticmethod
-    def _create_identifier(name, values):
-        identifier_type = IdentifierType.objects.get(name=name)
-        return Identifier(identifier_type=identifier_type, **dict(values))
+    def _create_identifier_args(values):
+        return {
+            "identifier_type": IdentifierType.objects.get(name=values[0]),
+            "nss": values[1],
+            "url": values[2],
+        }
+
+    _CREATE_DICT_FUNCTION_MAP = (
+        (Contributor, FKC_CONTRIBUTORS, _create_contributor_args),
+        (StoryArcNumber, FKC_STORY_ARC_NUMBERS, _create_story_arc_number_args),
+        (Identifier, FKC_IDENTIFIERS, _create_identifier_args),
+    )
 
-    @status_notify()
     def _bulk_create_dict_models(
-        self, create_tuples, create_method, model, status=None
+        self, create_tuples_key, create_args_func, model, status
     ):
         """Bulk create a dict type m2m model."""
+        create_tuples = self.metadata[FK_CREATE].pop(create_tuples_key, None)
         if not create_tuples:
-            return 0
+            return
 
         create_objs = []
-        for key, value in create_tuples:
-            obj = create_method(key, value)
+        for values_tuple in create_tuples:
+            args = create_args_func(values_tuple)
+            obj = model(**args)
             create_objs.append(obj)
 
         model.objects.bulk_create(
@@ -262,102 +299,44 @@ class CreateForeignKeysMixin(QueuedThread):
         if count:
             vnp = model._meta.verbose_name_plural.title()
             self.log.info(f"Created {count} {vnp}.")
-        if status:
-            status.complete = status.complete or 0
-            status.complete += count
-            self.status_controller.update(status)
-        return count
+        status.add_complete(count)
+        self.status_controller.update(status)
 
-    @staticmethod
-    def _get_create_fks_totals(create_data):
-        (
-            create_groups,
-            update_groups,
-            create_folder_paths,
-            create_fks,
-            create_contributors,
-            create_story_arc_numbers,
-            create_identifiers,
-        ) = create_data
-        total_fks = 0
-        for data_group in chain(
-            create_groups.values(), update_groups.values(), create_fks.values()
-        ):
-            total_fks += len(data_group)
-        total_fks += (
-            len(create_folder_paths)
-            + len(create_contributors)
-            + len(create_story_arc_numbers)
-            + len(create_identifiers)
-        )
-        return total_fks
-
-    def create_all_fks(self, library, create_data) -> int:
+    def create_all_fks(self):
         """Bulk create all foreign keys."""
-        total_fks = self._get_create_fks_totals(create_data)
-        status = Status(ImportStatusTypes.CREATE_FKS, 0, total_fks)
+        fkc = self.metadata.get(FK_CREATE)
+        if not fkc:
+            return
+
+        status = Status(ImportStatusTypes.CREATE_FKS, 0, fkc.pop(FKC_TOTAL_FKS, None))
         try:
             self.status_controller.start(status)
-            (
-                create_groups,
-                update_groups,
-                create_folder_paths,
-                create_fks,
-                create_contributors,
-                create_story_arc_numbers,
-                create_identifiers,
-            ) = create_data
-
-            for group_class, group_tree_counts in create_groups.items():
-                count = self._bulk_group_create(
-                    group_tree_counts,
-                    group_class,
-                    status=status,
-                )
-                status.add_complete(count)
 
-            for group_class, group_tree_counts in update_groups.items():
-                count = self._bulk_group_updater(
-                    group_tree_counts,
-                    group_class,
-                    status=status,
-                )
-                status.add_complete(count)
+            for group_class, group_tree_counts in fkc.pop(
+                FKC_CREATE_GROUPS, {}
+            ).items():
+                self._bulk_group_create(group_tree_counts, group_class, status)
 
-            count = self.bulk_folders_create(
-                sorted(create_folder_paths),
-                library,
-                status=status,
-            )
-            status.add_complete(count)
+            for group_class, group_tree_counts in fkc.pop(
+                FKC_UPDATE_GROUPS, {}
+            ).items():
+                self._bulk_group_updater(group_tree_counts, group_class, status)
 
-            for named_class, names in create_fks.items():
-                count = self._bulk_create_named_models(
-                    names,
-                    named_class,
-                    status=status,
-                )
-                status.add_complete(count)
+            self.bulk_folders_create(fkc.pop(FKC_FOLDER_PATHS, frozenset()), status)
+
+            for named_class, names in fkc.pop(FKC_CREATE_FKS, {}).items():
+                self._bulk_create_named_models(names, named_class, status)
 
             # These all depend on bulk_create_named_models running first
-            create_dict_data = (
-                (Contributor, create_contributors, self._create_contributor),
-                (
-                    StoryArcNumber,
-                    create_story_arc_numbers,
-                    self._create_story_arc_number,
-                ),
-                (Identifier, create_identifiers, self._create_identifier),
-            )
-            for model, create_objs, func in create_dict_data:
-                count = self._bulk_create_dict_models(
+            for model, create_objs, func in self._CREATE_DICT_FUNCTION_MAP:
+                self._bulk_create_dict_models(
                     create_objs,
                     func,
                     model,
-                    status=status,
+                    status,
                 )
-                status.add_complete(count)
 
         finally:
+            self.metadata.pop(FK_CREATE, None)
             self.status_controller.finish(status)
-        return status.complete if status.complete else 0
+        self.changed += status.complete if status.complete else 0