codex 1.6.0a6__py3-none-any.whl → 1.6.0a8__py3-none-any.whl

codex/librarian/importer/{clean_metadata.py → extract.py}

@@ -3,13 +3,18 @@
 from contextlib import suppress
 from decimal import ROUND_DOWN, Decimal
 from typing import Any
+from zipfile import BadZipFile
 
+from comicbox.box import Comicbox
 from comicbox.box.computed import IDENTIFIERS_KEY
+from comicbox.exceptions import UnsupportedArchiveTypeError
 from comicbox.schemas.comicbox_mixin import CONTRIBUTORS_KEY
 from django.db.models import CharField
 from django.db.models.fields import DecimalField, PositiveSmallIntegerField
+from rarfile import BadRarFile
 
-from codex.librarian.importer.const import STORY_ARCS_METADATA_KEY
+from codex.librarian.importer.const import FIS, STORY_ARCS_METADATA_KEY
+from codex.librarian.importer.query_fks import QueryForeignKeysImporter
 from codex.models import Comic
 from codex.models.named import (
     ContributorPerson,
@@ -18,7 +23,6 @@ from codex.models.named import (
     IdentifierType,
     StoryArc,
 )
-from codex.threads import QueuedThread
 
 _MD_INVALID_KEYS = frozenset(
     (
@@ -62,7 +66,7 @@ _DECIMAL_ZERO = Decimal("0.00")
 _ALPHA_2_LEN = 2
 
 
-class CleanMetadataMixin(QueuedThread):
+class ExtractMetadataImporter(QueryForeignKeysImporter):
     """Clean metadata before importing."""
 
     @staticmethod
@@ -219,7 +223,7 @@ class CleanMetadataMixin(QueuedThread):
         cls._assign_or_pop(md, IDENTIFIERS_KEY, clean_identifiers)
 
     @classmethod
-    def clean_md(cls, md):
+    def _clean_md(cls, md):
         """Sanitize the metadata before import."""
         cls._title_to_name(md)
         md = cls._prune_extra_keys(md)
@@ -231,3 +235,28 @@ class CleanMetadataMixin(QueuedThread):
         cls._clean_story_arcs(md)
         cls._clean_identifiers(md)
         return md
+
+    def extract_and_clean(self, path, import_metadata):
+        """Extract metadata from comic and clean it for codex."""
+        md = {}
+        failed_import = {}
+        try:
+            if import_metadata:
+                with Comicbox(path) as cb:
+                    md = cb.to_dict()
+                    md = md.get("comicbox", {})
+                    if "file_type" not in md:
+                        md["file_type"] = cb.get_file_type()
+                    if "page_count" not in md:
+                        md["page_count"] = cb.get_page_count()
+            md["path"] = path
+            md = self._clean_md(md)
+        except (UnsupportedArchiveTypeError, BadRarFile, BadZipFile, OSError) as exc:
+            self.log.warning(f"Failed to import {path}: {exc}")
+            failed_import = {path: exc}
+        except Exception as exc:
+            self.log.exception(f"Failed to import: {path}")
+            failed_import = {path: exc}
+        if failed_import:
+            self.metadata[FIS].update(failed_import)
+        return md

codex/librarian/importer/failed_imports.py

@@ -4,10 +4,11 @@ from pathlib import Path
 
 from django.db.models.functions import Now
 
-from codex.librarian.importer.status import ImportStatusTypes, status_notify
+from codex.librarian.importer.const import FIS
+from codex.librarian.importer.deleted import DeletedImporter
+from codex.librarian.importer.status import ImportStatusTypes
 from codex.models import Comic, FailedImport
 from codex.status import Status
-from codex.threads import QueuedThread
 
 _UPDATE_FIS = "update_fis"
 _CREATE_FIS = "create_fis"
@@ -15,20 +16,18 @@ _DELETE_FI_PATHS = "delete_fi_paths"
 _BULK_UPDATE_FAILED_IMPORT_FIELDS = ("name", "stat", "updated_at")
 
 
-class FailedImportsMixin(QueuedThread):
+class FailedImportsImporter(DeletedImporter):
     """Methods for failed imports."""
 
-    def _query_failed_import_deletes(
-        self, library, failed_imports, existing_failed_import_paths
-    ):
+    def _query_failed_import_deletes(self, existing_failed_import_paths):
         """Calculate Deletes."""
         untouched_failed_import_paths = existing_failed_import_paths - frozenset(
-            failed_imports.keys()
+            self.metadata[FIS].keys()
         )
 
         succeeded_failed_imports = frozenset(
             Comic.objects.filter(
-                library=library, path__in=untouched_failed_import_paths
+                library=self.library, path__in=untouched_failed_import_paths
             ).values_list("path", flat=True)
         )
 
@@ -42,53 +41,49 @@ class FailedImportsMixin(QueuedThread):
 
         return succeeded_failed_imports | missing_failed_imports
 
-    @status_notify()
     def _query_failed_imports(
         self,
-        failed_imports,
-        library,
         fis,
-        **kwargs,
     ):
         """Determine what to do with failed imports."""
-        if not failed_imports:
-            return 0
+        if not self.metadata[FIS]:
+            return
 
         # Remove the files deleted hack thing.
-        failed_imports.pop("files_deleted", None)
+        self.metadata[FIS].pop("files_deleted", None)
 
         existing_failed_import_paths = set(
-            FailedImport.objects.filter(library=library).values_list("path", flat=True)
+            FailedImport.objects.filter(library=self.library).values_list(
+                "path", flat=True
+            )
         )
 
         # Calculate creates and updates
-        for path, exc in failed_imports.items():
+        for path, exc in self.metadata[FIS].items():
             if path in existing_failed_import_paths:
                 fis[_UPDATE_FIS][path] = exc
             else:
                 fis[_CREATE_FIS][path] = exc
 
         fis[_DELETE_FI_PATHS] |= self._query_failed_import_deletes(
-            library, failed_imports, existing_failed_import_paths
+            existing_failed_import_paths
         )
+        self.metadata.pop(FIS)
 
         count = 0
         for iterable in fis.values():
             count += len(iterable)
-        return count
+        return
 
-    @status_notify()
-    def _bulk_update_failed_imports(
-        self, update_failed_imports, library, **kwargs
-    ) -> int:
+    def _bulk_update_failed_imports(self, update_failed_imports):
         """Bulk update failed imports."""
         if not update_failed_imports:
-            return 0
+            return
         update_failed_import_objs = FailedImport.objects.filter(
-            library=library, path__in=update_failed_imports.keys()
+            library=self.library, path__in=update_failed_imports.keys()
         ).only(*_BULK_UPDATE_FAILED_IMPORT_FIELDS)
         if not update_failed_import_objs:
-            return 0
+            return
         now = Now()
         for fi in update_failed_import_objs:
             try:
@@ -107,17 +102,16 @@ class FailedImportsMixin(QueuedThread):
         count = len(update_failed_import_objs)
         if count:
             self.log.info(f"Updated {count} old failed imports.")
-        return count
+        return
 
-    @status_notify()
-    def _bulk_create_failed_imports(self, create_failed_imports, library, **kwargs):
+    def _bulk_create_failed_imports(self, create_failed_imports):
         """Bulk create failed imports."""
         if not create_failed_imports:
             return 0
         create_objs = []
         for path, exc in create_failed_imports.items():
             try:
-                fi = FailedImport(library=library, path=path, parent_folder=None)
+                fi = FailedImport(library=self.library, path=path, parent_folder=None)
                 fi.set_reason(exc)
                 fi.presave()
                 create_objs.append(fi)
@@ -134,54 +128,39 @@ class FailedImportsMixin(QueuedThread):
             self.log.info(f"Added {count} comics to failed imports.")
         return count
 
-    @status_notify()
-    def _bulk_cleanup_failed_imports(
-        self, delete_failed_imports_paths, library, **kwargs
-    ):
+    def _bulk_cleanup_failed_imports(self, delete_failed_imports_paths):
         """Remove FailedImport objects that have since succeeded."""
         if not delete_failed_imports_paths:
             return 0
         # Cleanup FailedImports that were actually successful
-        qs = FailedImport.objects.filter(library=library).filter(
+        qs = FailedImport.objects.filter(library=self.library).filter(
             path__in=delete_failed_imports_paths
         )
         qs.delete()
         count = len(delete_failed_imports_paths)
         if count:
-            self.log.info(f"Cleaned up {count} failed imports from {library.path}")
+            self.log.info(f"Cleaned up {count} failed imports from {self.library.path}")
         return count
 
-    def fail_imports(self, library, failed_imports, is_files_deleted):
+    def fail_imports(self):
         """Handle failed imports."""
+        fis = self.metadata.get(FIS)
+        if not fis:
+            return False
+        status = Status(ImportStatusTypes.FAILED_IMPORTS)
+        self.status_controller.start(status)
         created_count = 0
+
         try:
             fis = {_UPDATE_FIS: {}, _CREATE_FIS: {}, _DELETE_FI_PATHS: set()}
-            if is_files_deleted:
+            if self.task.files_deleted:
                 # if any files were deleted. Run the failed import check
-                failed_imports["files_deleted"] = True
-            status = Status(ImportStatusTypes.FAILED_IMPORTS, 0, len(failed_imports))
-            status.total = self._query_failed_imports(
-                failed_imports,
-                library,
-                fis,
-                status=status,
-            )
-
-            self._bulk_update_failed_imports(
-                fis[_UPDATE_FIS],
-                library,
-                status=status,
-            )
-
-            created_count = self._bulk_create_failed_imports(
-                fis[_CREATE_FIS],
-                library,
-                status=status,
-            )
-
-            self._bulk_cleanup_failed_imports(
-                fis[_DELETE_FI_PATHS], library, status=status
-            )
+                self.metadata[FIS]["files_deleted"] = True
+            self._query_failed_imports(fis)
+            self._bulk_update_failed_imports(fis[_UPDATE_FIS])
+            self._bulk_create_failed_imports(fis[_CREATE_FIS])
+            self._bulk_cleanup_failed_imports(fis[_DELETE_FI_PATHS])
         except Exception:
             self.log.exception("Processing failed imports")
+        self.status_controller.finish(status)
         return bool(created_count)

codex/librarian/importer/importer.py

@@ -0,0 +1,96 @@
+"""The main importer class."""
+
+from time import time
+
+from django.core.cache import cache
+from humanize import naturaldelta
+
+from codex.librarian.importer.moved import MovedImporter
+from codex.librarian.importer.status import ImportStatusTypes
+from codex.librarian.notifier.tasks import FAILED_IMPORTS_TASK, LIBRARY_CHANGED_TASK
+from codex.librarian.search.tasks import SearchIndexUpdateTask
+from codex.librarian.tasks import DelayedTasks
+
+
+class ComicImporter(MovedImporter):
+    """Initialize, run and finish a bulk import."""
+
+    ##########
+    # FINISH #
+    ##########
+    def _finish_apply_status(self):
+        """Finish all librarian statuses."""
+        self.library.update_in_progress = False
+        self.library.save()
+        self.status_controller.finish_many(ImportStatusTypes.values)
+
+    def _finish_apply(self, new_failed_imports, imported_count):
+        """Perform final tasks when the apply is done."""
+        if self.changed:
+            cache.clear()
+            elapsed_time = time() - self.start_time.timestamp()
+            elapsed = naturaldelta(elapsed_time)
+            log_txt = f"Updated library {self.library.path} in {elapsed}."
+            if imported_count:
+                cps = round(imported_count / elapsed_time, 1)
+                log_txt += (
+                    f" Imported {imported_count} comics at {cps} comics per second."
+                )
+            else:
+                log_txt += " No comics to import."
+            self.log.info(log_txt)
+            self.librarian_queue.put(LIBRARY_CHANGED_TASK)
+
+            # Wait to start the search index update in case more updates are incoming.
+            until = time() + 3
+            delayed_search_task = DelayedTasks(until, (SearchIndexUpdateTask(False),))
+            self.librarian_queue.put(delayed_search_task)
+        else:
+            self.log.info("No updates neccissary.")
+        if new_failed_imports:
+            self.librarian_queue.put(FAILED_IMPORTS_TASK)
+
+    def apply(self):
+        """Bulk import comics."""
+        try:
+            self.init_apply()
+            self.move_and_modify_dirs()
+
+            #############
+            # AGGREGATE #
+            #############
+            self.get_aggregate_metadata()
+
+            #########
+            # QUERY #
+            #########
+            self.query_all_missing_fks()
+            self.query_missing_custom_covers()
+
+            #####################
+            # UPDATE AND CREATE #
+            #####################
+            self.create_all_fks()
+            self.update_custom_covers()
+            self.create_custom_covers()
+
+            imported_count = self.bulk_update_comics()
+            imported_count += self.bulk_create_comics()
+
+            ########
+            # LINK #
+            ########
+            self.bulk_query_and_link_comic_m2m_fields()
+            self.link_custom_covers()
+            self.changed += imported_count
+
+            new_failed_imports = self.fail_imports()
+
+            deleted_comic_groups = self.delete()
+
+            self.update_all_groups_first_comics(deleted_comic_groups, self.start_time)
+
+        finally:
+            self._finish_apply_status()
+
+        self._finish_apply(new_failed_imports, imported_count)