codex 1.6.0a6__py3-none-any.whl → 1.6.0a8__py3-none-any.whl

codex/librarian/importer/create_covers.py

@@ -0,0 +1,98 @@
+"""Create Custom Covers."""
+
+from django.db.models.functions.datetime import Now
+
+from codex.librarian.importer.const import (
+    COVERS_CREATE,
+    COVERS_UPDATE,
+    CUSTOM_COVER_UPDATE_FIELDS,
+    LINK_COVER_PKS,
+)
+from codex.librarian.importer.create_comics import CreateComicsImporter
+from codex.librarian.importer.status import ImportStatusTypes
+from codex.models import (
+    CustomCover,
+)
+from codex.status import Status
+
+
+class CreateCoversImporter(CreateComicsImporter):
+    """Create Custom Covers."""
+
+    def update_custom_covers(self, status=None):
+        """Update Custom Covers."""
+        update_covers_qs = self.metadata.pop(COVERS_UPDATE, None)
+        if not update_covers_qs:
+            return
+        update_covers_count = update_covers_qs.count()
+        if not update_covers_count:
+            return
+        status = Status(ImportStatusTypes.COVERS_MODIFIED, 0, update_covers_count)
+        self.status_controller.start(status)
+
+        now = Now()
+
+        update_covers = []
+        for cover in update_covers_qs.only(*CUSTOM_COVER_UPDATE_FIELDS):
+            cover.updated_at = now
+            cover.presave()
+            update_covers.append(cover)
+
+        count = 0
+        if update_covers:
+            CustomCover.objects.bulk_update(update_covers, CUSTOM_COVER_UPDATE_FIELDS)
+            update_cover_pks = update_covers_qs.values_list("pk", flat=True)
+            if LINK_COVER_PKS not in self.metadata:
+                self.metadata[LINK_COVER_PKS] = set()
+            self.metadata[LINK_COVER_PKS].update(update_cover_pks)
+            self._remove_covers(update_cover_pks, custom=True)  # type: ignore
+            count = len(update_covers)
+            if status:
+                status.add_complete(count)
+
+            link_covers_status = Status(
+                ImportStatusTypes.COVERS_LINK, 0, len(self.metadata[LINK_COVER_PKS])
+            )
+            self.status_controller.update(link_covers_status, notify=False)
+
+        self.changed += count
+        self.status_controller.finish(status)
+
+    def create_custom_covers(self, status=None):
+        """Create Custom Covers."""
+        create_cover_paths = self.metadata.pop(COVERS_CREATE, ())
+        num_create_cover_paths = len(create_cover_paths)
+        if not num_create_cover_paths:
+            return
+        status = Status(ImportStatusTypes.COVERS_MODIFIED, 0, num_create_cover_paths)
+        self.status_controller.start(status)
+
+        create_covers = []
+        for path in create_cover_paths:
+            cover = CustomCover(library=self.library, path=path)
+            cover.presave()
+            create_covers.append(cover)
+
+        count = 0
+        if create_covers:
+            objs = CustomCover.objects.bulk_create(
+                create_covers,
+                update_conflicts=True,
+                update_fields=("path", "stat"),
+                unique_fields=CustomCover._meta.unique_together[0],
+            )
+            created_pks = frozenset(obj.pk for obj in objs)
+            if LINK_COVER_PKS not in self.metadata:
+                self.metadata[LINK_COVER_PKS] = set()
+            self.metadata[LINK_COVER_PKS].update(created_pks)
+            count = len(created_pks)
+            if status:
+                status.add_complete(count)
+
+            link_covers_status = Status(
+                ImportStatusTypes.COVERS_LINK, 0, len(self.metadata[LINK_COVER_PKS])
+            )
+            self.status_controller.update(link_covers_status, notify=False)
+
+        self.changed += count
+        self.status_controller.finish(status)

codex/librarian/importer/create_fks.py

@@ -6,14 +6,21 @@ So we may safely create the comics next.
 from pathlib import Path
 
 from django.core.exceptions import ObjectDoesNotExist
-from django.db.models.functions.datetime import Now
 
 from codex.librarian.importer.const import (
     BULK_UPDATE_FOLDER_FIELDS,
     CLASS_CUSTOM_COVER_GROUP_MAP,
     COUNT_FIELDS,
     CREATE_DICT_UPDATE_FIELDS,
-    CUSTOM_COVER_UPDATE_FIELDS,
+    FK_CREATE,
+    FKC_CONTRIBUTORS,
+    FKC_CREATE_FKS,
+    FKC_CREATE_GROUPS,
+    FKC_FOLDER_PATHS,
+    FKC_IDENTIFIERS,
+    FKC_STORY_ARC_NUMBERS,
+    FKC_TOTAL_FKS,
+    FKC_UPDATE_GROUPS,
     GROUP_BASE_FIELDS,
     GROUP_UPDATE_FIELDS,
     IMPRINT,
@@ -23,7 +30,8 @@ from codex.librarian.importer.const import (
     SERIES,
     VOLUME_COUNT,
 )
-from codex.librarian.importer.status import ImportStatusTypes, status_notify
+from codex.librarian.importer.create_covers import CreateCoversImporter
+from codex.librarian.importer.status import ImportStatusTypes
 from codex.models import (
     Contributor,
     ContributorPerson,
@@ -39,10 +47,9 @@ from codex.models import (
 )
 from codex.models.named import Identifier, IdentifierType
 from codex.status import Status
-from codex.threads import QueuedThread
 
 
-class CreateForeignKeysMixin(QueuedThread):
+class CreateForeignKeysImporter(CreateCoversImporter):
     """Methods for creating foreign keys."""
 
     @staticmethod
@@ -107,12 +114,11 @@ class CreateForeignKeysMixin(QueuedThread):
             obj = None
         return obj
 
-    @status_notify()
-    def _bulk_group_create(self, group_tree_counts, group_class, status=None) -> int:
+    def _bulk_group_create(self, group_tree_counts, group_class, status):
         """Bulk creates groups."""
         count = 0
         if not group_tree_counts:
-            return count
+            return
         create_groups = []
         for group_param_tuple, group_count in group_tree_counts.items():
             obj = self._create_group_obj(group_class, group_param_tuple, group_count)
@@ -130,18 +136,14 @@ class CreateForeignKeysMixin(QueuedThread):
         if count:
             vnp = group_class._meta.verbose_name_plural.title()
             self.log.info(f"Created {count} {vnp}.")
-        if status:
-            status.complete = status.complete or 0
-            status.complete += count
-            self.status_controller.update(status)
-        return count
+        status.add_complete(count)
+        self.status_controller.update(status)
+        return
 
-    @status_notify()
-    def _bulk_group_updater(self, group_tree_counts, group_class, status=None):
+    def _bulk_group_updater(self, group_tree_counts, group_class, status):
         """Bulk update groups."""
-        count = 0
         if not group_tree_counts:
-            return count
+            return
         update_groups = []
         for group_param_tuple, group_count in group_tree_counts.items():
             obj = self._update_group_obj(group_class, group_param_tuple, group_count)
@@ -149,30 +151,27 @@ class CreateForeignKeysMixin(QueuedThread):
                 update_groups.append(obj)
         count_field = COUNT_FIELDS[group_class]
         group_class.objects.bulk_update(update_groups, fields=[count_field])
-        count += len(update_groups)
+        count = len(update_groups)
         if count:
             vnp = group_class._meta.verbose_name_plural.title()
             self.log.info(f"Updated {count} {vnp}.")
-        if status:
-            status.complete = status.complete or 0
-            status.complete += count
-            self.status_controller.update(status)
-        return count
+        status.add_complete(count)
+        self.status_controller.update(status)
 
-    def _bulk_folders_create_add_folder(self, library, path, create_folders):
+    def _bulk_folders_create_add_folder(self, path, create_folders):
         """Add one folder to the create list."""
         parent_path = str(path.parent)
         parent = None
         try:
             parent = Folder.objects.get(path=parent_path)
         except Folder.DoesNotExist:
-            if path.parent != Path(library.path):
+            if path.parent != Path(self.library.path):
                 self.log.warning(
                     f"Can't find parent folder {parent_path}"
-                    f" for {path} in library {library.path}"
+                    f" for {path} in library {self.library.path}"
                 )
         folder = Folder(
-            library=library,
+            library=self.library,
             path=str(path),
             name=path.name,
             parent_folder=parent,
@@ -181,11 +180,11 @@ class CreateForeignKeysMixin(QueuedThread):
         self._add_custom_cover_to_group(Folder, folder)
         create_folders.append(folder)
 
-    def _bulk_folders_create_depth_level(self, library, paths, status):
+    def _bulk_folders_create_depth_level(self, paths, status):
         """Create a depth level of folders."""
         create_folders = []
         for path in sorted(paths):
-            self._bulk_folders_create_add_folder(library, path, create_folders)
+            self._bulk_folders_create_add_folder(path, create_folders)
         Folder.objects.bulk_create(
             create_folders,
             update_conflicts=True,
@@ -193,18 +192,15 @@ class CreateForeignKeysMixin(QueuedThread):
             unique_fields=Folder._meta.unique_together[0],  # type: ignore
         )
         count = len(create_folders)
-        if status:
-            status.complete = status.complete or 0
-            status.complete += count
-            self.status_controller.update(status)
+        status.add_complete(count)
+        self.status_controller.update(status)
         return count
 
-    @status_notify()
-    def bulk_folders_create(self, folder_paths: set, library, status=None):
+    def bulk_folders_create(self, folder_paths: frozenset, status):
         """Create folders breadth first."""
         count = 0
         if not folder_paths:
-            return count
+            return
         # group folder paths by depth
         folder_path_dict = {}
         for path_str in folder_paths:
@@ -215,19 +211,18 @@ class CreateForeignKeysMixin(QueuedThread):
             folder_path_dict[path_length].add(path)
 
         # create each depth level first to ensure we can assign parents
-        for _, paths in sorted(folder_path_dict.items()):
-            count += self._bulk_folders_create_depth_level(library, paths, status)
+        for paths in sorted(folder_path_dict.values()):
+            count += self._bulk_folders_create_depth_level(paths, status)
 
         if count:
             self.log.info(f"Created {count} Folders.")
-        return count
+        self.status_controller.update(status)
 
-    @status_notify()
-    def _bulk_create_named_models(self, names, named_class, status=None):
+    def _bulk_create_named_models(self, names, named_class, status):
         """Bulk create named models."""
         count = len(names)
         if not count:
-            return count
+            return
         create_named_objs = []
         is_story_arc = named_class == StoryArc
         for name in names:
@@ -250,11 +245,8 @@ class CreateForeignKeysMixin(QueuedThread):
         if count:
             vnp = named_class._meta.verbose_name_plural.title()
             self.log.info(f"Created {count} {vnp}.")
-        if status:
-            status.complete = status.complete or 0
-            status.complete += count
-            self.status_controller.update(status)
-        return count
+        status.add_complete(count)
+        self.status_controller.update(status)
 
     @staticmethod
     def _create_contributor_args(values):
@@ -275,17 +267,23 @@ class CreateForeignKeysMixin(QueuedThread):
             "url": values[2],
         }
 
-    @status_notify()
+    _CREATE_DICT_FUNCTION_MAP = (
+        (Contributor, FKC_CONTRIBUTORS, _create_contributor_args),
+        (StoryArcNumber, FKC_STORY_ARC_NUMBERS, _create_story_arc_number_args),
+        (Identifier, FKC_IDENTIFIERS, _create_identifier_args),
+    )
+
     def _bulk_create_dict_models(
-        self, create_tuples, create_args_method, model, status=None
+        self, create_tuples_key, create_args_func, model, status
     ):
         """Bulk create a dict type m2m model."""
+        create_tuples = self.metadata[FK_CREATE].pop(create_tuples_key, None)
         if not create_tuples:
-            return 0
+            return
 
         create_objs = []
         for values_tuple in create_tuples:
-            args = create_args_method(values_tuple)
+            args = create_args_func(values_tuple)
             obj = model(**args)
             create_objs.append(obj)
 
@@ -299,139 +297,44 @@ class CreateForeignKeysMixin(QueuedThread):
         if count:
             vnp = model._meta.verbose_name_plural.title()
             self.log.info(f"Created {count} {vnp}.")
-        if status:
-            status.complete = status.complete or 0
-            status.complete += count
-            self.status_controller.update(status)
-        return count
+        status.add_complete(count)
+        self.status_controller.update(status)
 
-    def create_all_fks(self, library, create_data) -> int:
+    def create_all_fks(self):
         """Bulk create all foreign keys."""
-        (
-            create_groups,
-            update_groups,
-            create_folder_paths,
-            create_fks,
-            create_contributors,
-            create_story_arc_numbers,
-            create_identifiers,
-            total_fks,
-        ) = create_data
-
-        status = Status(ImportStatusTypes.CREATE_FKS, 0, total_fks)
+        fkc = self.metadata.get(FK_CREATE)
+        if not fkc:
+            return
+
+        status = Status(ImportStatusTypes.CREATE_FKS, 0, fkc.pop(FKC_TOTAL_FKS, None))
         try:
             self.status_controller.start(status)
 
-            for group_class, group_tree_counts in create_groups.items():
-                count = self._bulk_group_create(
-                    group_tree_counts,
-                    group_class,
-                    status=status,
-                )
-                status.add_complete(count)
+            for group_class, group_tree_counts in fkc.pop(
+                FKC_CREATE_GROUPS, {}
+            ).items():
+                self._bulk_group_create(group_tree_counts, group_class, status)
 
-            for group_class, group_tree_counts in update_groups.items():
-                count = self._bulk_group_updater(
-                    group_tree_counts,
-                    group_class,
-                    status=status,
-                )
-                status.add_complete(count)
+            for group_class, group_tree_counts in fkc.pop(
+                FKC_UPDATE_GROUPS, {}
+            ).items():
+                self._bulk_group_updater(group_tree_counts, group_class, status)
 
-            count = self.bulk_folders_create(
-                sorted(create_folder_paths),
-                library,
-                status=status,
-            )
-            status.add_complete(count)
+            self.bulk_folders_create(fkc.pop(FKC_FOLDER_PATHS, frozenset()), status)
 
-            for named_class, names in create_fks.items():
-                count = self._bulk_create_named_models(
-                    names,
-                    named_class,
-                    status=status,
-                )
-                status.add_complete(count)
+            for named_class, names in fkc.pop(FKC_CREATE_FKS, {}).items():
+                self._bulk_create_named_models(names, named_class, status)
 
             # These all depend on bulk_create_named_models running first
-            create_dict_data = (
-                (Contributor, create_contributors, self._create_contributor_args),
-                (
-                    StoryArcNumber,
-                    create_story_arc_numbers,
-                    self._create_story_arc_number_args,
-                ),
-                (Identifier, create_identifiers, self._create_identifier_args),
-            )
-            for model, create_objs, func in create_dict_data:
-                count = self._bulk_create_dict_models(
+            for model, create_objs, func in self._CREATE_DICT_FUNCTION_MAP:
+                self._bulk_create_dict_models(
                     create_objs,
                     func,
                     model,
-                    status=status,
+                    status,
                 )
-                status.add_complete(count)
 
         finally:
+            self.metadata.pop(FK_CREATE, None)
             self.status_controller.finish(status)
-        return status.complete if status.complete else 0
-
-    @status_notify(ImportStatusTypes.COVERS_MODIFIED, updates=False)
-    def update_custom_covers(
-        self, update_covers_qs, link_cover_pks, status=None
-    ) -> int:
-        """Update Custom Covers."""
-        count = 0
-        update_covers_count = update_covers_qs.count()
-        if not update_covers_count:
-            return count
-        if status:
-            status.total = update_covers_count
-        now = Now()
-
-        update_covers = []
-        for cover in update_covers_qs.only(*CUSTOM_COVER_UPDATE_FIELDS):
-            cover.updated_at = now
-            cover.presave()
-            update_covers.append(cover)
-
-        if update_covers:
-            CustomCover.objects.bulk_update(update_covers, CUSTOM_COVER_UPDATE_FIELDS)
-            update_cover_pks = update_covers_qs.values_list("pk", flat=True)
-            link_cover_pks.update(update_cover_pks)
-            self._remove_covers(update_cover_pks, custom=True)  # type: ignore
-            count = len(update_covers)
-            if status:
-                status.add_complete(count)
-        return count
-
-    @status_notify(ImportStatusTypes.COVERS_CREATED, updates=False)
-    def create_custom_covers(
-        self, create_cover_paths, library, link_cover_pks, status=None
-    ) -> int:
-        """Create Custom Covers."""
-        count = 0
-        if not create_cover_paths:
-            return count
-        if status:
-            status.total = len(create_cover_paths)
-
-        create_covers = []
-        for path in create_cover_paths:
-            cover = CustomCover(library=library, path=path)
-            cover.presave()
-            create_covers.append(cover)
-
-        if create_covers:
-            objs = CustomCover.objects.bulk_create(
-                create_covers,
-                update_conflicts=True,
-                update_fields=("path", "stat"),
-                unique_fields=CustomCover._meta.unique_together[0],
-            )
-            created_pks = frozenset(obj.pk for obj in objs)
-            link_cover_pks.update(created_pks)
-            count = len(created_pks)
-            if status:
-                status.add_complete(count)
-        return count
+        self.changed += status.complete if status.complete else 0

codex/librarian/importer/deleted.py

@@ -1,29 +1,34 @@
 """Clean up the database after moves or imports."""
 
 from codex.librarian.covers.tasks import CoverRemoveTask
+from codex.librarian.importer.cache import CacheUpdateImporter
 from codex.librarian.importer.const import COMIC_GROUP_FIELD_NAMES
-from codex.librarian.importer.status import ImportStatusTypes, status_notify
+from codex.librarian.importer.status import ImportStatusTypes
 from codex.models import Comic, Folder, StoryArc
 from codex.models.paths import CustomCover
 from codex.settings.settings import MAX_CHUNK_SIZE
-from codex.threads import QueuedThread
+from codex.status import Status
 
 
-class DeletedMixin(QueuedThread):
+class DeletedImporter(CacheUpdateImporter):
     """Clean up database methods."""
 
     def _remove_covers(self, delete_pks, custom=False):
         task = CoverRemoveTask(delete_pks, custom)
         self.librarian_queue.put(task)
 
-    @status_notify(status_type=ImportStatusTypes.DIRS_DELETED, updates=False)
-    def _bulk_folders_deleted(self, delete_folder_paths, library, **kwargs):
+    def _bulk_folders_deleted(self, **kwargs):
         """Bulk delete folders."""
-        if not delete_folder_paths:
+        if not self.task.dirs_deleted:
             return 0
-        folders = Folder.objects.filter(library=library, path__in=delete_folder_paths)
+        status = Status(ImportStatusTypes.DIRS_DELETED, 0, len(self.task.dirs_deleted))
+        self.status_controller.start(status)
+        folders = Folder.objects.filter(
+            library=self.library, path__in=self.task.dirs_deleted
+        )
+        self.task.dirs_deleted = frozenset()
         delete_comic_pks = frozenset(
-            Comic.objects.filter(library=library, folders__in=folders)
+            Comic.objects.filter(library=self.library, folders__in=folders)
             .distinct()
             .values_list("pk", flat=True)
         )
@@ -31,14 +36,27 @@ class DeletedMixin(QueuedThread):
 
         self._remove_covers(delete_comic_pks)
 
-        count = len(delete_folder_paths)
+        count = len(delete_comic_pks)
         if count:
             self.log.info(
                 f"Deleted {count} folders and {len(delete_comic_pks)} comics"
-                f"from {library.path}"
+                f"from {self.library.path}"
             )
+        self.status_controller.finish(status)
         return count
 
+    @staticmethod
+    def _init_deleted_comic_groups():
+        """Init deleted_comic_groups, used later even if no deletes."""
+        deleted_comic_groups = {}
+        for field_name in COMIC_GROUP_FIELD_NAMES:
+            if field_name == "story_arc_numbers":
+                related_model = StoryArc
+            else:
+                related_model = Comic._meta.get_field(field_name).related_model
+            deleted_comic_groups[related_model] = set()
+        return deleted_comic_groups
+
     @staticmethod
     def _populate_deleted_comic_groups(delete_qs, deleted_comic_groups):
         """Populate changed groups for cover timestamp updater."""
@@ -60,21 +78,18 @@ class DeletedMixin(QueuedThread):
                     related_id = getattr(comic, field_name).pk
                     deleted_comic_groups[related_model].add(related_id)
 
-    @staticmethod
-    def _init_deleted_comic_groups(deleted_comic_groups):
-        """Init deleted_comic_groups, used later even if no deletes."""
-        for field_name in COMIC_GROUP_FIELD_NAMES:
-            related_model = Comic._meta.get_field(field_name).related_model
-            deleted_comic_groups[related_model] = set()
-
-    @status_notify(status_type=ImportStatusTypes.FILES_DELETED, updates=False)
-    def _bulk_comics_deleted(
-        self, delete_comic_paths, library, deleted_comic_groups, **kwargs
-    ):
+    def _bulk_comics_deleted(self, deleted_comic_groups, **kwargs):
         """Bulk delete comics found missing from the filesystem."""
-        if not delete_comic_paths:
+        if not self.task.files_deleted:
             return 0
-        delete_qs = Comic.objects.filter(library=library, path__in=delete_comic_paths)
+        status = Status(
+            ImportStatusTypes.FILES_DELETED, 0, len(self.task.files_deleted)
+        )
+        self.status_controller.start(status)
+        delete_qs = Comic.objects.filter(
+            library=self.library, path__in=self.task.files_deleted
+        )
+        self.task.files_deleted = frozenset()
 
         self._populate_deleted_comic_groups(delete_qs, deleted_comic_groups)
 
@@ -83,43 +98,41 @@ class DeletedMixin(QueuedThread):
 
         self._remove_covers(delete_comic_pks)
 
-        count = len(delete_comic_paths)
+        count = len(delete_comic_pks)
         if count:
-            self.log.info(f"Deleted {count} comics from {library.path}")
-
+            self.log.info(f"Deleted {count} comics from {self.library.path}")
+        self.status_controller.finish(status)
         return count
 
-    @status_notify(status_type=ImportStatusTypes.COVERS_DELETED, updates=False)
-    def _bulk_covers_deleted(self, delete_cover_paths, library, **kwargs):
+    def _bulk_covers_deleted(self, **kwargs):
         """Bulk delete comics found missing from the filesystem."""
-        if not delete_cover_paths:
+        if not self.task.covers_deleted:
             return 0
+        status = Status(
+            ImportStatusTypes.COVERS_DELETED, 0, len(self.task.covers_deleted)
+        )
+        self.status_controller.start(status)
         covers = CustomCover.objects.filter(
-            library=library, path__in=delete_cover_paths
+            library=self.library, path__in=self.task.covers_deleted
         )
+        self.task.covers_deleted = frozenset()
         delete_cover_pks = frozenset(covers.values_list("pk", flat=True))
         covers.delete()
 
         self._remove_covers(delete_cover_pks, custom=True)
 
-        count = len(delete_cover_paths)
+        count = len(delete_cover_pks)
         if count:
-            self.log.info(f"Deleted {count} custom covers from {library.path}")
+            self.log.info(f"Deleted {count} custom covers from {self.library.path}")
 
+        self.status_controller.finish(status)
         return count
 
-    def delete(self, library, task, deleted_comic_groups):
+    def delete(self):
         """Delete files and folders."""
-        count = self._bulk_folders_deleted(task.dirs_deleted, library)
-        task.dirs_deleted = None
-
-        self._init_deleted_comic_groups(deleted_comic_groups)
-        count += self._bulk_comics_deleted(
-            task.files_deleted, library, deleted_comic_groups
-        )
-        task.files_deleted = None
-
-        count += self._bulk_covers_deleted(task.covers_deleted, library)
-        task.covers_deleted = None
-
-        return count
+        count = self._bulk_folders_deleted()
+        deleted_comic_groups = self._init_deleted_comic_groups()
+        count += self._bulk_comics_deleted(deleted_comic_groups)
+        count += self._bulk_covers_deleted()
+        self.changed += count
+        return deleted_comic_groups