codex 1.6.0__py3-none-any.whl → 1.6.0a0__py3-none-any.whl

codex/librarian/importer/create_fks.py

@@ -3,24 +3,13 @@
 So we may safely create the comics next.
 """
 
+from itertools import chain
 from pathlib import Path
 
-from django.core.exceptions import ObjectDoesNotExist
-
 from codex.librarian.importer.const import (
     BULK_UPDATE_FOLDER_FIELDS,
-    CLASS_CUSTOM_COVER_GROUP_MAP,
     COUNT_FIELDS,
     CREATE_DICT_UPDATE_FIELDS,
-    FK_CREATE,
-    FKC_CONTRIBUTORS,
-    FKC_CREATE_FKS,
-    FKC_CREATE_GROUPS,
-    FKC_FOLDER_PATHS,
-    FKC_IDENTIFIERS,
-    FKC_STORY_ARC_NUMBERS,
-    FKC_TOTAL_FKS,
-    FKC_UPDATE_GROUPS,
     GROUP_BASE_FIELDS,
     GROUP_UPDATE_FIELDS,
     IMPRINT,
@@ -30,13 +19,11 @@ from codex.librarian.importer.const import (
     SERIES,
     VOLUME_COUNT,
 )
-from codex.librarian.importer.create_covers import CreateCoversImporter
-from codex.librarian.importer.status import ImportStatusTypes
+from codex.librarian.importer.status import ImportStatusTypes, status_notify
 from codex.models import (
     Contributor,
     ContributorPerson,
     ContributorRole,
-    CustomCover,
     Folder,
     Imprint,
     Publisher,
@@ -47,26 +34,14 @@ from codex.models import (
 )
 from codex.models.named import Identifier, IdentifierType
 from codex.status import Status
+from codex.threads import QueuedThread
 
 
-class CreateForeignKeysImporter(CreateCoversImporter):
+class CreateForeignKeysMixin(QueuedThread):
     """Methods for creating foreign keys."""
 
     @staticmethod
-    def _add_custom_cover_to_group(group_class, obj):
-        """If a custom cover exists for this group, add it."""
-        group = CLASS_CUSTOM_COVER_GROUP_MAP.get(group_class)
-        if not group:
-            # Normal, volume doesn't link to covers
-            return
-        try:
-            cover = CustomCover.objects.filter(group=group, sort_name=obj.sort_name)[0]
-            obj.custom_cover = cover
-        except (IndexError, ObjectDoesNotExist):
-            pass
-
-    @classmethod
-    def _create_group_obj(cls, group_class, group_param_tuple, group_count):
+    def _create_group_obj(group_class, group_param_tuple, group_count):
         """Create a set of browser group objects."""
         defaults = {"name": group_param_tuple[-1]}
         if group_class in (Imprint, Series, Volume):
@@ -89,7 +64,6 @@ class CreateForeignKeysImporter(CreateCoversImporter):
 
         obj = group_class(**defaults)
         obj.presave()
-        cls._add_custom_cover_to_group(group_class, obj)
         return obj
 
     @staticmethod
@@ -114,18 +88,17 @@ class CreateForeignKeysImporter(CreateCoversImporter):
             obj = None
         return obj
 
-    def _bulk_group_create(self, group_tree_counts, group_class, status):
+    @status_notify()
+    def _bulk_group_create(self, group_tree_counts, group_class, status=None) -> int:
         """Bulk creates groups."""
         count = 0
         if not group_tree_counts:
-            return
+            return count
         create_groups = []
         for group_param_tuple, group_count in group_tree_counts.items():
             obj = self._create_group_obj(group_class, group_param_tuple, group_count)
             create_groups.append(obj)
         update_fields = GROUP_UPDATE_FIELDS[group_class]
-        if group_class in CLASS_CUSTOM_COVER_GROUP_MAP:
-            update_fields += ("custom_cover",)
         group_class.objects.bulk_create(
             create_groups,
             update_conflicts=True,
@@ -136,14 +109,18 @@ class CreateForeignKeysImporter(CreateCoversImporter):
         if count:
             vnp = group_class._meta.verbose_name_plural.title()
             self.log.info(f"Created {count} {vnp}.")
-        status.add_complete(count)
-        self.status_controller.update(status)
-        return
+        if status:
+            status.complete = status.complete or 0
+            status.complete += count
+            self.status_controller.update(status)
+        return count
 
-    def _bulk_group_updater(self, group_tree_counts, group_class, status):
+    @status_notify()
+    def _bulk_group_updater(self, group_tree_counts, group_class, status=None):
         """Bulk update groups."""
+        count = 0
         if not group_tree_counts:
-            return
+            return count
         update_groups = []
         for group_param_tuple, group_count in group_tree_counts.items():
             obj = self._update_group_obj(group_class, group_param_tuple, group_count)
@@ -151,40 +128,42 @@ class CreateForeignKeysImporter(CreateCoversImporter):
                 update_groups.append(obj)
         count_field = COUNT_FIELDS[group_class]
         group_class.objects.bulk_update(update_groups, fields=[count_field])
-        count = len(update_groups)
+        count += len(update_groups)
         if count:
             vnp = group_class._meta.verbose_name_plural.title()
             self.log.info(f"Updated {count} {vnp}.")
-        status.add_complete(count)
-        self.status_controller.update(status)
+        if status:
+            status.complete = status.complete or 0
+            status.complete += count
+            self.status_controller.update(status)
+        return count
 
-    def _bulk_folders_create_add_folder(self, path, create_folders):
+    def _bulk_folders_create_add_folder(self, library, path, create_folders):
         """Add one folder to the create list."""
         parent_path = str(path.parent)
         parent = None
         try:
             parent = Folder.objects.get(path=parent_path)
         except Folder.DoesNotExist:
-            if path.parent != Path(self.library.path):
+            if path.parent != Path(library.path):
                 self.log.warning(
                     f"Can't find parent folder {parent_path}"
-                    f" for {path} in library {self.library.path}"
+                    f" for {path} in library {library.path}"
                 )
         folder = Folder(
-            library=self.library,
+            library=library,
             path=str(path),
             name=path.name,
             parent_folder=parent,
         )
         folder.presave()
-        self._add_custom_cover_to_group(Folder, folder)
         create_folders.append(folder)
 
-    def _bulk_folders_create_depth_level(self, paths, status):
+    def _bulk_folders_create_depth_level(self, library, paths, status):
         """Create a depth level of folders."""
         create_folders = []
         for path in sorted(paths):
-            self._bulk_folders_create_add_folder(path, create_folders)
+            self._bulk_folders_create_add_folder(library, path, create_folders)
         Folder.objects.bulk_create(
             create_folders,
             update_conflicts=True,
@@ -192,15 +171,18 @@ class CreateForeignKeysImporter(CreateCoversImporter):
             unique_fields=Folder._meta.unique_together[0],  # type: ignore
         )
         count = len(create_folders)
-        status.add_complete(count)
-        self.status_controller.update(status)
+        if status:
+            status.complete = status.complete or 0
+            status.complete += count
+            self.status_controller.update(status)
         return count
 
-    def bulk_folders_create(self, folder_paths: frozenset, status):
+    @status_notify()
+    def bulk_folders_create(self, folder_paths: set, library, status=None):
         """Create folders breadth first."""
         count = 0
         if not folder_paths:
-            return
+            return count
         # group folder paths by depth
         folder_path_dict = {}
         for path_str in folder_paths:
@@ -211,27 +193,25 @@ class CreateForeignKeysImporter(CreateCoversImporter):
             folder_path_dict[path_length].add(path)
 
         # create each depth level first to ensure we can assign parents
-        for depth_level in sorted(folder_path_dict):
-            paths = sorted(folder_path_dict[depth_level])
-            count += self._bulk_folders_create_depth_level(paths, status)
-            self.log.debug(f"Created {count} Folders at depth level {depth_level}")
+        for _, paths in sorted(folder_path_dict.items()):
+            count += self._bulk_folders_create_depth_level(library, paths, status)
 
         if count:
             self.log.info(f"Created {count} Folders.")
-        self.status_controller.update(status)
+        return count
 
-    def _bulk_create_named_models(self, names, named_class, status):
+    @status_notify()
+    def _bulk_create_named_models(self, names, named_class, status=None):
         """Bulk create named models."""
         count = len(names)
         if not count:
-            return
+            return count
         create_named_objs = []
         is_story_arc = named_class == StoryArc
         for name in names:
             named_obj = named_class(name=name)
             if is_story_arc:
                 named_obj.presave()
-                self._add_custom_cover_to_group(StoryArc, named_obj)
             create_named_objs.append(named_obj)
 
         update_fields = NAMED_MODEL_UPDATE_FIELDS
@@ -247,8 +227,11 @@ class CreateForeignKeysImporter(CreateCoversImporter):
         if count:
             vnp = named_class._meta.verbose_name_plural.title()
             self.log.info(f"Created {count} {vnp}.")
-        status.add_complete(count)
-        self.status_controller.update(status)
+        if status:
+            status.complete = status.complete or 0
+            status.complete += count
+            self.status_controller.update(status)
+        return count
 
     @staticmethod
     def _create_contributor_args(values):
@@ -269,23 +252,17 @@ class CreateForeignKeysImporter(CreateCoversImporter):
             "url": values[2],
         }
 
-    _CREATE_DICT_FUNCTION_MAP = (
-        (Contributor, FKC_CONTRIBUTORS, _create_contributor_args),
-        (StoryArcNumber, FKC_STORY_ARC_NUMBERS, _create_story_arc_number_args),
-        (Identifier, FKC_IDENTIFIERS, _create_identifier_args),
-    )
-
+    @status_notify()
     def _bulk_create_dict_models(
-        self, create_tuples_key, create_args_func, model, status
+        self, create_tuples, create_args_method, model, status=None
     ):
         """Bulk create a dict type m2m model."""
-        create_tuples = self.metadata[FK_CREATE].pop(create_tuples_key, None)
         if not create_tuples:
-            return
+            return 0
 
         create_objs = []
         for values_tuple in create_tuples:
-            args = create_args_func(values_tuple)
+            args = create_args_method(values_tuple)
             obj = model(**args)
             create_objs.append(obj)
 
@@ -299,44 +276,102 @@ class CreateForeignKeysImporter(CreateCoversImporter):
         if count:
             vnp = model._meta.verbose_name_plural.title()
             self.log.info(f"Created {count} {vnp}.")
-        status.add_complete(count)
-        self.status_controller.update(status)
+        if status:
+            status.complete = status.complete or 0
+            status.complete += count
+            self.status_controller.update(status)
+        return count
 
-    def create_all_fks(self):
-        """Bulk create all foreign keys."""
-        fkc = self.metadata.get(FK_CREATE)
-        if not fkc:
-            return
+    @staticmethod
+    def _get_create_fks_totals(create_data):
+        (
+            create_groups,
+            update_groups,
+            create_folder_paths,
+            create_fks,
+            create_contributors,
+            create_story_arc_numbers,
+            create_identifiers,
+        ) = create_data
+        total_fks = 0
+        for data_group in chain(
+            create_groups.values(), update_groups.values(), create_fks.values()
+        ):
+            total_fks += len(data_group)
+        total_fks += (
+            len(create_folder_paths)
+            + len(create_contributors)
+            + len(create_story_arc_numbers)
+            + len(create_identifiers)
+        )
+        return total_fks
 
-        status = Status(ImportStatusTypes.CREATE_FKS, 0, fkc.pop(FKC_TOTAL_FKS, None))
+    def create_all_fks(self, library, create_data) -> int:
+        """Bulk create all foreign keys."""
+        total_fks = self._get_create_fks_totals(create_data)
+        status = Status(ImportStatusTypes.CREATE_FKS, 0, total_fks)
         try:
             self.status_controller.start(status)
+            (
+                create_groups,
+                update_groups,
+                create_folder_paths,
+                create_fks,
+                create_contributors,
+                create_story_arc_numbers,
+                create_identifiers,
+            ) = create_data
+
+            for group_class, group_tree_counts in create_groups.items():
+                count = self._bulk_group_create(
+                    group_tree_counts,
+                    group_class,
+                    status=status,
+                )
+                status.add_complete(count)
 
-            for group_class, group_tree_counts in fkc.pop(
-                FKC_CREATE_GROUPS, {}
-            ).items():
-                self._bulk_group_create(group_tree_counts, group_class, status)
-
-            for group_class, group_tree_counts in fkc.pop(
-                FKC_UPDATE_GROUPS, {}
-            ).items():
-                self._bulk_group_updater(group_tree_counts, group_class, status)
+            for group_class, group_tree_counts in update_groups.items():
+                count = self._bulk_group_updater(
+                    group_tree_counts,
+                    group_class,
+                    status=status,
+                )
+                status.add_complete(count)
 
-            self.bulk_folders_create(fkc.pop(FKC_FOLDER_PATHS, frozenset()), status)
+            count = self.bulk_folders_create(
+                sorted(create_folder_paths),
+                library,
+                status=status,
+            )
+            status.add_complete(count)
 
-            for named_class, names in fkc.pop(FKC_CREATE_FKS, {}).items():
-                self._bulk_create_named_models(names, named_class, status)
+            for named_class, names in create_fks.items():
+                count = self._bulk_create_named_models(
+                    names,
+                    named_class,
+                    status=status,
+                )
+                status.add_complete(count)
 
             # These all depend on bulk_create_named_models running first
-            for model, create_objs, func in self._CREATE_DICT_FUNCTION_MAP:
-                self._bulk_create_dict_models(
+            create_dict_data = (
+                (Contributor, create_contributors, self._create_contributor_args),
+                (
+                    StoryArcNumber,
+                    create_story_arc_numbers,
+                    self._create_story_arc_number_args,
+                ),
+                (Identifier, create_identifiers, self._create_identifier_args),
+            )
+            for model, create_objs, func in create_dict_data:
+                count = self._bulk_create_dict_models(
                     create_objs,
                     func,
                     model,
-                    status,
+                    status=status,
                 )
+                status.add_complete(count)
 
         finally:
-            self.metadata.pop(FK_CREATE, None)
             self.status_controller.finish(status)
-        self.changed += status.complete if status.complete else 0
+        return status.complete if status.complete else 0

codex/librarian/importer/deleted.py

@@ -1,34 +1,26 @@
 """Clean up the database after moves or imports."""
 
 from codex.librarian.covers.tasks import CoverRemoveTask
-from codex.librarian.importer.cache import CacheUpdateImporter
-from codex.librarian.importer.const import COMIC_GROUP_FIELD_NAMES
-from codex.librarian.importer.status import ImportStatusTypes
-from codex.models import Comic, Folder, StoryArc
-from codex.models.paths import CustomCover
-from codex.settings.settings import MAX_CHUNK_SIZE
-from codex.status import Status
+from codex.librarian.importer.status import ImportStatusTypes, status_notify
+from codex.models import Comic, Folder
+from codex.threads import QueuedThread
 
 
-class DeletedImporter(CacheUpdateImporter):
+class DeletedMixin(QueuedThread):
     """Clean up database methods."""
 
-    def _remove_covers(self, delete_pks, custom=False):
-        task = CoverRemoveTask(delete_pks, custom)
+    def _remove_covers(self, delete_comic_pks):
+        task = CoverRemoveTask(delete_comic_pks)
         self.librarian_queue.put(task)
 
-    def _bulk_folders_deleted(self, **kwargs):
+    @status_notify(status_type=ImportStatusTypes.DIRS_DELETED, updates=False)
+    def _bulk_folders_deleted(self, delete_folder_paths, library, **kwargs):
         """Bulk delete folders."""
-        if not self.task.dirs_deleted:
+        if not delete_folder_paths:
             return 0
-        status = Status(ImportStatusTypes.DIRS_DELETED, 0, len(self.task.dirs_deleted))
-        self.status_controller.start(status)
-        folders = Folder.objects.filter(
-            library=self.library, path__in=self.task.dirs_deleted
-        )
-        self.task.dirs_deleted = frozenset()
+        folders = Folder.objects.filter(library=library, path__in=delete_folder_paths)
         delete_comic_pks = frozenset(
-            Comic.objects.filter(library=self.library, folders__in=folders)
+            Comic.objects.filter(library=library, folders__in=folders)
             .distinct()
             .values_list("pk", flat=True)
         )
@@ -36,103 +28,36 @@ class DeletedImporter(CacheUpdateImporter):
 
         self._remove_covers(delete_comic_pks)
 
-        count = len(delete_comic_pks)
+        count = len(delete_folder_paths)
         if count:
             self.log.info(
                 f"Deleted {count} folders and {len(delete_comic_pks)} comics"
-                f"from {self.library.path}"
+                f"from {library.path}"
             )
-        self.status_controller.finish(status)
         return count
 
-    @staticmethod
-    def _init_deleted_comic_groups():
-        """Init deleted_comic_groups, used later even if no deletes."""
-        deleted_comic_groups = {}
-        for field_name in COMIC_GROUP_FIELD_NAMES:
-            if field_name == "story_arc_numbers":
-                related_model = StoryArc
-            else:
-                related_model = Comic._meta.get_field(field_name).related_model
-            deleted_comic_groups[related_model] = set()
-        return deleted_comic_groups
-
-    @staticmethod
-    def _populate_deleted_comic_groups(delete_qs, deleted_comic_groups):
-        """Populate changed groups for cover timestamp updater."""
-        comics_deleted_qs = delete_qs.only(*COMIC_GROUP_FIELD_NAMES).prefetch_related(
-            "story_arc_numbers__story_arc"
-        )
-        for comic in comics_deleted_qs.iterator(chunk_size=MAX_CHUNK_SIZE):
-            for field_name in COMIC_GROUP_FIELD_NAMES:
-                if field_name == "story_arc_numbers":
-                    for san in comic.story_arc_numbers.select_related("story_arc").only(
-                        "story_arc"
-                    ):
-                        deleted_comic_groups[StoryArc].add(san.story_arc.pk)
-                elif field_name == "folders":
-                    for folder in comic.folders.only("pk"):
-                        deleted_comic_groups[Folder].add(folder.pk)
-                else:
-                    related_model = comic._meta.get_field(field_name).related_model
-                    related_id = getattr(comic, field_name).pk
-                    deleted_comic_groups[related_model].add(related_id)
-
-    def _bulk_comics_deleted(self, deleted_comic_groups, **kwargs):
+    @status_notify(status_type=ImportStatusTypes.FILES_DELETED, updates=False)
+    def _bulk_comics_deleted(self, delete_comic_paths, library, **kwargs):
         """Bulk delete comics found missing from the filesystem."""
-        if not self.task.files_deleted:
+        if not delete_comic_paths:
             return 0
-        status = Status(
-            ImportStatusTypes.FILES_DELETED, 0, len(self.task.files_deleted)
-        )
-        self.status_controller.start(status)
-        delete_qs = Comic.objects.filter(
-            library=self.library, path__in=self.task.files_deleted
-        )
-        self.task.files_deleted = frozenset()
-
-        self._populate_deleted_comic_groups(delete_qs, deleted_comic_groups)
-
-        delete_comic_pks = frozenset(delete_qs.values_list("pk", flat=True))
-        delete_qs.delete()
+        comics = Comic.objects.filter(library=library, path__in=delete_comic_paths)
+        delete_comic_pks = frozenset(comics.values_list("pk", flat=True))
+        comics.delete()
 
         self._remove_covers(delete_comic_pks)
 
-        count = len(delete_comic_pks)
+        count = len(delete_comic_paths)
         if count:
-            self.log.info(f"Deleted {count} comics from {self.library.path}")
-        self.status_controller.finish(status)
-        return count
+            self.log.info(f"Deleted {count} comics from {library.path}")
 
-    def _bulk_covers_deleted(self, **kwargs):
-        """Bulk delete comics found missing from the filesystem."""
-        if not self.task.covers_deleted:
-            return 0
-        status = Status(
-            ImportStatusTypes.COVERS_DELETED, 0, len(self.task.covers_deleted)
-        )
-        self.status_controller.start(status)
-        covers = CustomCover.objects.filter(
-            library=self.library, path__in=self.task.covers_deleted
-        )
-        self.task.covers_deleted = frozenset()
-        delete_cover_pks = frozenset(covers.values_list("pk", flat=True))
-        covers.delete()
-
-        self._remove_covers(delete_cover_pks, custom=True)
-
-        count = len(delete_cover_pks)
-        if count:
-            self.log.info(f"Deleted {count} custom covers from {self.library.path}")
-
-        self.status_controller.finish(status)
         return count
 
-    def delete(self):
+    def delete(self, library, task):
         """Delete files and folders."""
-        count = self._bulk_folders_deleted()
-        deleted_comic_groups = self._init_deleted_comic_groups()
-        count += self._bulk_comics_deleted(deleted_comic_groups)
-        count += self._bulk_covers_deleted()
-        self.changed += count
-        return deleted_comic_groups
+        count = self._bulk_folders_deleted(task.dirs_deleted, library)
+        task.dirs_deleted = None
+
+        count += self._bulk_comics_deleted(task.files_deleted, library)
+        task.files_deleted = None
+        return count