codex 1.5.19__py3-none-any.whl → 1.6.0__py3-none-any.whl

codex/librarian/importer/deleted.py

@@ -1,26 +1,34 @@
 """Clean up the database after moves or imports."""
 
 from codex.librarian.covers.tasks import CoverRemoveTask
-from codex.librarian.importer.status import ImportStatusTypes, status_notify
-from codex.models import Comic, Folder
-from codex.threads import QueuedThread
+from codex.librarian.importer.cache import CacheUpdateImporter
+from codex.librarian.importer.const import COMIC_GROUP_FIELD_NAMES
+from codex.librarian.importer.status import ImportStatusTypes
+from codex.models import Comic, Folder, StoryArc
+from codex.models.paths import CustomCover
+from codex.settings.settings import MAX_CHUNK_SIZE
+from codex.status import Status
 
 
-class DeletedMixin(QueuedThread):
+class DeletedImporter(CacheUpdateImporter):
     """Clean up database methods."""
 
-    def _remove_covers(self, delete_comic_pks):
-        task = CoverRemoveTask(delete_comic_pks)
+    def _remove_covers(self, delete_pks, custom=False):
+        task = CoverRemoveTask(delete_pks, custom)
         self.librarian_queue.put(task)
 
-    @status_notify(status_type=ImportStatusTypes.DIRS_DELETED, updates=False)
-    def _bulk_folders_deleted(self, delete_folder_paths, library, **kwargs):
+    def _bulk_folders_deleted(self, **kwargs):
         """Bulk delete folders."""
-        if not delete_folder_paths:
+        if not self.task.dirs_deleted:
             return 0
-        folders = Folder.objects.filter(library=library, path__in=delete_folder_paths)
+        status = Status(ImportStatusTypes.DIRS_DELETED, 0, len(self.task.dirs_deleted))
+        self.status_controller.start(status)
+        folders = Folder.objects.filter(
+            library=self.library, path__in=self.task.dirs_deleted
+        )
+        self.task.dirs_deleted = frozenset()
         delete_comic_pks = frozenset(
-            Comic.objects.filter(library=library, folders__in=folders)
+            Comic.objects.filter(library=self.library, folders__in=folders)
             .distinct()
             .values_list("pk", flat=True)
         )
@@ -28,36 +36,103 @@ class DeletedMixin(QueuedThread):
 
         self._remove_covers(delete_comic_pks)
 
-        count = len(delete_folder_paths)
+        count = len(delete_comic_pks)
         if count:
             self.log.info(
                 f"Deleted {count} folders and {len(delete_comic_pks)} comics"
-                f"from {library.path}"
+                f"from {self.library.path}"
             )
+        self.status_controller.finish(status)
         return count
 
-    @status_notify(status_type=ImportStatusTypes.FILES_DELETED, updates=False)
-    def _bulk_comics_deleted(self, delete_comic_paths, library, **kwargs):
+    @staticmethod
+    def _init_deleted_comic_groups():
+        """Init deleted_comic_groups, used later even if no deletes."""
+        deleted_comic_groups = {}
+        for field_name in COMIC_GROUP_FIELD_NAMES:
+            if field_name == "story_arc_numbers":
+                related_model = StoryArc
+            else:
+                related_model = Comic._meta.get_field(field_name).related_model
+            deleted_comic_groups[related_model] = set()
+        return deleted_comic_groups
+
+    @staticmethod
+    def _populate_deleted_comic_groups(delete_qs, deleted_comic_groups):
+        """Populate changed groups for cover timestamp updater."""
+        comics_deleted_qs = delete_qs.only(*COMIC_GROUP_FIELD_NAMES).prefetch_related(
+            "story_arc_numbers__story_arc"
+        )
+        for comic in comics_deleted_qs.iterator(chunk_size=MAX_CHUNK_SIZE):
+            for field_name in COMIC_GROUP_FIELD_NAMES:
+                if field_name == "story_arc_numbers":
+                    for san in comic.story_arc_numbers.select_related("story_arc").only(
+                        "story_arc"
+                    ):
+                        deleted_comic_groups[StoryArc].add(san.story_arc.pk)
+                elif field_name == "folders":
+                    for folder in comic.folders.only("pk"):
+                        deleted_comic_groups[Folder].add(folder.pk)
+                else:
+                    related_model = comic._meta.get_field(field_name).related_model
+                    related_id = getattr(comic, field_name).pk
+                    deleted_comic_groups[related_model].add(related_id)
+
+    def _bulk_comics_deleted(self, deleted_comic_groups, **kwargs):
         """Bulk delete comics found missing from the filesystem."""
-        if not delete_comic_paths:
+        if not self.task.files_deleted:
             return 0
-        comics = Comic.objects.filter(library=library, path__in=delete_comic_paths)
-        delete_comic_pks = frozenset(comics.values_list("pk", flat=True))
-        comics.delete()
+        status = Status(
+            ImportStatusTypes.FILES_DELETED, 0, len(self.task.files_deleted)
+        )
+        self.status_controller.start(status)
+        delete_qs = Comic.objects.filter(
+            library=self.library, path__in=self.task.files_deleted
+        )
+        self.task.files_deleted = frozenset()
+
+        self._populate_deleted_comic_groups(delete_qs, deleted_comic_groups)
+
+        delete_comic_pks = frozenset(delete_qs.values_list("pk", flat=True))
+        delete_qs.delete()
 
         self._remove_covers(delete_comic_pks)
 
-        count = len(delete_comic_paths)
+        count = len(delete_comic_pks)
         if count:
-            self.log.info(f"Deleted {count} comics from {library.path}")
-
+            self.log.info(f"Deleted {count} comics from {self.library.path}")
+        self.status_controller.finish(status)
         return count
 
-    def delete(self, library, task):
-        """Delete files and folders."""
-        count = self._bulk_folders_deleted(task.dirs_deleted, library)
-        task.dirs_deleted = None
+    def _bulk_covers_deleted(self, **kwargs):
+        """Bulk delete comics found missing from the filesystem."""
+        if not self.task.covers_deleted:
+            return 0
+        status = Status(
+            ImportStatusTypes.COVERS_DELETED, 0, len(self.task.covers_deleted)
+        )
+        self.status_controller.start(status)
+        covers = CustomCover.objects.filter(
+            library=self.library, path__in=self.task.covers_deleted
+        )
+        self.task.covers_deleted = frozenset()
+        delete_cover_pks = frozenset(covers.values_list("pk", flat=True))
+        covers.delete()
+
+        self._remove_covers(delete_cover_pks, custom=True)
 
-        count += self._bulk_comics_deleted(task.files_deleted, library)
-        task.files_deleted = None
+        count = len(delete_cover_pks)
+        if count:
+            self.log.info(f"Deleted {count} custom covers from {self.library.path}")
+
+        self.status_controller.finish(status)
         return count
+
+    def delete(self):
+        """Delete files and folders."""
+        count = self._bulk_folders_deleted()
+        deleted_comic_groups = self._init_deleted_comic_groups()
+        count += self._bulk_comics_deleted(deleted_comic_groups)
+        count += self._bulk_covers_deleted()
+        self.changed += count
+        return deleted_comic_groups

codex/librarian/importer/{clean_metadata.py → extract.py}

@@ -3,13 +3,18 @@
 from contextlib import suppress
 from decimal import ROUND_DOWN, Decimal
 from typing import Any
+from zipfile import BadZipFile
 
+from comicbox.box import Comicbox
 from comicbox.box.computed import IDENTIFIERS_KEY
+from comicbox.exceptions import UnsupportedArchiveTypeError
 from comicbox.schemas.comicbox_mixin import CONTRIBUTORS_KEY
 from django.db.models import CharField
 from django.db.models.fields import DecimalField, PositiveSmallIntegerField
+from rarfile import BadRarFile
 
-from codex.librarian.importer.const import STORY_ARCS_METADATA_KEY
+from codex.librarian.importer.const import FIS, STORY_ARCS_METADATA_KEY
+from codex.librarian.importer.query_fks import QueryForeignKeysImporter
 from codex.models import Comic
 from codex.models.named import (
     ContributorPerson,
@@ -18,7 +23,6 @@ from codex.models.named import (
     IdentifierType,
     StoryArc,
 )
-from codex.threads import QueuedThread
 
 _MD_INVALID_KEYS = frozenset(
     (
@@ -62,7 +66,7 @@ _DECIMAL_ZERO = Decimal("0.00")
 _ALPHA_2_LEN = 2
 
 
-class CleanMetadataMixin(QueuedThread):
+class ExtractMetadataImporter(QueryForeignKeysImporter):
     """Clean metadata before importing."""
 
     @staticmethod
@@ -219,7 +223,7 @@ class CleanMetadataMixin(QueuedThread):
         cls._assign_or_pop(md, IDENTIFIERS_KEY, clean_identifiers)
 
     @classmethod
-    def clean_md(cls, md):
+    def _clean_md(cls, md):
         """Sanitize the metadata before import."""
         cls._title_to_name(md)
         md = cls._prune_extra_keys(md)
@@ -231,3 +235,28 @@ class CleanMetadataMixin(QueuedThread):
         cls._clean_story_arcs(md)
         cls._clean_identifiers(md)
         return md
+
+    def extract_and_clean(self, path, import_metadata):
+        """Extract metadata from comic and clean it for codex."""
+        md = {}
+        failed_import = {}
+        try:
+            if import_metadata:
+                with Comicbox(path) as cb:
+                    md = cb.to_dict()
+                    md = md.get("comicbox", {})
+                    if "file_type" not in md:
+                        md["file_type"] = cb.get_file_type()
+                    if "page_count" not in md:
+                        md["page_count"] = cb.get_page_count()
+            md["path"] = path
+            md = self._clean_md(md)
+        except (UnsupportedArchiveTypeError, BadRarFile, BadZipFile, OSError) as exc:
+            self.log.warning(f"Failed to import {path}: {exc}")
+            failed_import = {path: exc}
+        except Exception as exc:
+            self.log.exception(f"Failed to import: {path}")
+            failed_import = {path: exc}
+        if failed_import:
+            self.metadata[FIS].update(failed_import)
+        return md

codex/librarian/importer/failed_imports.py

@@ -4,10 +4,11 @@ from pathlib import Path
 
 from django.db.models.functions import Now
 
-from codex.librarian.importer.status import ImportStatusTypes, status_notify
+from codex.librarian.importer.const import FIS
+from codex.librarian.importer.deleted import DeletedImporter
+from codex.librarian.importer.status import ImportStatusTypes
 from codex.models import Comic, FailedImport
 from codex.status import Status
-from codex.threads import QueuedThread
 
 _UPDATE_FIS = "update_fis"
 _CREATE_FIS = "create_fis"
@@ -15,20 +16,18 @@ _DELETE_FI_PATHS = "delete_fi_paths"
 _BULK_UPDATE_FAILED_IMPORT_FIELDS = ("name", "stat", "updated_at")
 
 
-class FailedImportsMixin(QueuedThread):
+class FailedImportsImporter(DeletedImporter):
     """Methods for failed imports."""
 
-    def _query_failed_import_deletes(
-        self, library, failed_imports, existing_failed_import_paths
-    ):
+    def _query_failed_import_deletes(self, existing_failed_import_paths):
         """Calculate Deletes."""
         untouched_failed_import_paths = existing_failed_import_paths - frozenset(
-            failed_imports.keys()
+            self.metadata[FIS].keys()
         )
 
         succeeded_failed_imports = frozenset(
             Comic.objects.filter(
-                library=library, path__in=untouched_failed_import_paths
+                library=self.library, path__in=untouched_failed_import_paths
             ).values_list("path", flat=True)
         )
 
@@ -42,60 +41,55 @@ class FailedImportsMixin(QueuedThread):
 
         return succeeded_failed_imports | missing_failed_imports
 
-    @status_notify()
     def _query_failed_imports(
         self,
-        failed_imports,
-        library,
         fis,
-        **kwargs,
     ):
         """Determine what to do with failed imports."""
-        if not failed_imports:
-            return 0
+        if not self.metadata[FIS]:
+            return
 
         # Remove the files deleted hack thing.
-        failed_imports.pop("files_deleted", None)
+        self.metadata[FIS].pop("files_deleted", None)
 
         existing_failed_import_paths = set(
-            FailedImport.objects.filter(library=library).values_list("path", flat=True)
+            FailedImport.objects.filter(library=self.library).values_list(
+                "path", flat=True
+            )
         )
 
         # Calculate creates and updates
-        for path, exc in failed_imports.items():
+        for path, exc in self.metadata[FIS].items():
             if path in existing_failed_import_paths:
                 fis[_UPDATE_FIS][path] = exc
             else:
                 fis[_CREATE_FIS][path] = exc
 
         fis[_DELETE_FI_PATHS] |= self._query_failed_import_deletes(
-            library, failed_imports, existing_failed_import_paths
+            existing_failed_import_paths
         )
+        self.metadata.pop(FIS)
 
         count = 0
         for iterable in fis.values():
             count += len(iterable)
-        return count
+        return
 
-    @status_notify()
-    def _bulk_update_failed_imports(
-        self, update_failed_imports, library, **kwargs
-    ) -> int:
+    def _bulk_update_failed_imports(self, update_failed_imports):
         """Bulk update failed imports."""
         if not update_failed_imports:
-            return 0
+            return
         update_failed_import_objs = FailedImport.objects.filter(
-            library=library, path__in=update_failed_imports.keys()
+            library=self.library, path__in=update_failed_imports.keys()
         ).only(*_BULK_UPDATE_FAILED_IMPORT_FIELDS)
         if not update_failed_import_objs:
-            return 0
-        now = Now()
+            return
         for fi in update_failed_import_objs:
             try:
                 exc = update_failed_imports.pop(fi.path)
                 fi.set_reason(exc)
-                fi.set_stat()
-                fi.updated_at = now
+                fi.presave()
+                fi.updated_at = Now()
             except Exception as exc:
                 self.log.exception(
                     f"Error preparing failed import update for {fi.path}"
@@ -107,19 +101,18 @@ class FailedImportsMixin(QueuedThread):
         count = len(update_failed_import_objs)
         if count:
             self.log.info(f"Updated {count} old failed imports.")
-        return count
+        return
 
-    @status_notify()
-    def _bulk_create_failed_imports(self, create_failed_imports, library, **kwargs):
+    def _bulk_create_failed_imports(self, create_failed_imports):
         """Bulk create failed imports."""
         if not create_failed_imports:
             return 0
         create_objs = []
         for path, exc in create_failed_imports.items():
             try:
-                fi = FailedImport(library=library, path=path, parent_folder=None)
+                fi = FailedImport(library=self.library, path=path, parent_folder=None)
                 fi.set_reason(exc)
-                fi.set_stat()
+                fi.presave()
                 create_objs.append(fi)
             except Exception as exc:
                 self.log.exception(f"Error preparing failed import create for {path}")
@@ -134,54 +127,39 @@ class FailedImportsMixin(QueuedThread):
             self.log.info(f"Added {count} comics to failed imports.")
         return count
 
-    @status_notify()
-    def _bulk_cleanup_failed_imports(
-        self, delete_failed_imports_paths, library, **kwargs
-    ):
+    def _bulk_cleanup_failed_imports(self, delete_failed_imports_paths):
         """Remove FailedImport objects that have since succeeded."""
         if not delete_failed_imports_paths:
             return 0
         # Cleanup FailedImports that were actually successful
-        qs = FailedImport.objects.filter(library=library).filter(
+        qs = FailedImport.objects.filter(library=self.library).filter(
             path__in=delete_failed_imports_paths
         )
         qs.delete()
         count = len(delete_failed_imports_paths)
         if count:
-            self.log.info(f"Cleaned up {count} failed imports from {library.path}")
+            self.log.info(f"Cleaned up {count} failed imports from {self.library.path}")
         return count
 
-    def fail_imports(self, library, failed_imports, is_files_deleted):
+    def fail_imports(self):
         """Handle failed imports."""
+        fis = self.metadata.get(FIS)
+        if not fis:
+            return False
+        status = Status(ImportStatusTypes.FAILED_IMPORTS)
+        self.status_controller.start(status)
         created_count = 0
+
         try:
             fis = {_UPDATE_FIS: {}, _CREATE_FIS: {}, _DELETE_FI_PATHS: set()}
-            if is_files_deleted:
+            if self.task.files_deleted:
                 # if any files were deleted. Run the failed import check
-                failed_imports["files_deleted"] = True
-            status = Status(ImportStatusTypes.FAILED_IMPORTS, 0, len(failed_imports))
-            status.total = self._query_failed_imports(
-                failed_imports,
-                library,
-                fis,
-                status=status,
-            )
-
-            self._bulk_update_failed_imports(
-                fis[_UPDATE_FIS],
-                library,
-                status=status,
-            )
-
-            created_count = self._bulk_create_failed_imports(
-                fis[_CREATE_FIS],
-                library,
-                status=status,
-            )
-
-            self._bulk_cleanup_failed_imports(
-                fis[_DELETE_FI_PATHS], library, status=status
-            )
+                self.metadata[FIS]["files_deleted"] = True
+            self._query_failed_imports(fis)
+            self._bulk_update_failed_imports(fis[_UPDATE_FIS])
+            self._bulk_create_failed_imports(fis[_CREATE_FIS])
+            self._bulk_cleanup_failed_imports(fis[_DELETE_FI_PATHS])
         except Exception:
             self.log.exception("Processing failed imports")
+        self.status_controller.finish(status)
         return bool(created_count)

codex/librarian/importer/importer.py

@@ -0,0 +1,96 @@
+"""The main importer class."""
+
+from time import time
+
+from django.core.cache import cache
+from humanize import naturaldelta
+
+from codex.librarian.importer.moved import MovedImporter
+from codex.librarian.importer.status import ImportStatusTypes
+from codex.librarian.notifier.tasks import FAILED_IMPORTS_TASK, LIBRARY_CHANGED_TASK
+from codex.librarian.search.tasks import SearchIndexUpdateTask
+from codex.librarian.tasks import DelayedTasks
+
+
+class ComicImporter(MovedImporter):
+    """Initialize, run and finish a bulk import."""
+
+    ##########
+    # FINISH #
+    ##########
+    def _finish_apply_status(self):
+        """Finish all librarian statuses."""
+        self.library.update_in_progress = False
+        self.library.save()
+        self.status_controller.finish_many(ImportStatusTypes.values)
+
+    def _finish_apply(self, new_failed_imports, imported_count):
+        """Perform final tasks when the apply is done."""
+        if self.changed:
+            cache.clear()
+            elapsed_time = time() - self.start_time.timestamp()
+            elapsed = naturaldelta(elapsed_time)
+            log_txt = f"Updated library {self.library.path} in {elapsed}."
+            if imported_count:
+                cps = round(imported_count / elapsed_time, 1)
+                log_txt += (
+                    f" Imported {imported_count} comics at {cps} comics per second."
+                )
+            else:
+                log_txt += " No comics to import."
+            self.log.info(log_txt)
+            self.librarian_queue.put(LIBRARY_CHANGED_TASK)
+
+            # Wait to start the search index update in case more updates are incoming.
+            until = time() + 3
+            delayed_search_task = DelayedTasks(until, (SearchIndexUpdateTask(False),))
+            self.librarian_queue.put(delayed_search_task)
+        else:
+            self.log.info("No updates neccissary.")
+        if new_failed_imports:
+            self.librarian_queue.put(FAILED_IMPORTS_TASK)
+
+    def apply(self):
+        """Bulk import comics."""
+        try:
+            self.init_apply()
+            self.move_and_modify_dirs()
+
+            #############
+            # AGGREGATE #
+            #############
+            self.get_aggregate_metadata()
+
+            #########
+            # QUERY #
+            #########
+            self.query_all_missing_fks()
+            self.query_missing_custom_covers()
+
+            #####################
+            # UPDATE AND CREATE #
+            #####################
+            self.create_all_fks()
+            self.update_custom_covers()
+            self.create_custom_covers()
+
+            imported_count = self.bulk_update_comics()
+            imported_count += self.bulk_create_comics()
+
+            ########
+            # LINK #
+            ########
+            self.bulk_query_and_link_comic_m2m_fields()
+            self.link_custom_covers()
+            self.changed += imported_count
+
+            new_failed_imports = self.fail_imports()
+
+            deleted_comic_groups = self.delete()
+
+            self.update_all_groups(deleted_comic_groups, self.start_time)
+
+        finally:
+            self._finish_apply_status()
+
+        self._finish_apply(new_failed_imports, imported_count)