codex 1.6.0__py3-none-any.whl → 1.6.0a1__py3-none-any.whl

codex/librarian/importer/failed_imports.py

@@ -4,11 +4,10 @@ from pathlib import Path
 
 from django.db.models.functions import Now
 
-from codex.librarian.importer.const import FIS
-from codex.librarian.importer.deleted import DeletedImporter
-from codex.librarian.importer.status import ImportStatusTypes
+from codex.librarian.importer.status import ImportStatusTypes, status_notify
 from codex.models import Comic, FailedImport
 from codex.status import Status
+from codex.threads import QueuedThread
 
 _UPDATE_FIS = "update_fis"
 _CREATE_FIS = "create_fis"
@@ -16,18 +15,20 @@ _DELETE_FI_PATHS = "delete_fi_paths"
 _BULK_UPDATE_FAILED_IMPORT_FIELDS = ("name", "stat", "updated_at")
 
 
-class FailedImportsImporter(DeletedImporter):
+class FailedImportsMixin(QueuedThread):
     """Methods for failed imports."""
 
-    def _query_failed_import_deletes(self, existing_failed_import_paths):
+    def _query_failed_import_deletes(
+        self, library, failed_imports, existing_failed_import_paths
+    ):
         """Calculate Deletes."""
         untouched_failed_import_paths = existing_failed_import_paths - frozenset(
-            self.metadata[FIS].keys()
+            failed_imports.keys()
         )
 
         succeeded_failed_imports = frozenset(
             Comic.objects.filter(
-                library=self.library, path__in=untouched_failed_import_paths
+                library=library, path__in=untouched_failed_import_paths
             ).values_list("path", flat=True)
         )
 
@@ -41,55 +42,60 @@ class FailedImportsImporter(DeletedImporter):
 
         return succeeded_failed_imports | missing_failed_imports
 
+    @status_notify()
     def _query_failed_imports(
         self,
+        failed_imports,
+        library,
         fis,
+        **kwargs,
     ):
         """Determine what to do with failed imports."""
-        if not self.metadata[FIS]:
-            return
+        if not failed_imports:
+            return 0
 
         # Remove the files deleted hack thing.
-        self.metadata[FIS].pop("files_deleted", None)
+        failed_imports.pop("files_deleted", None)
 
         existing_failed_import_paths = set(
-            FailedImport.objects.filter(library=self.library).values_list(
-                "path", flat=True
-            )
+            FailedImport.objects.filter(library=library).values_list("path", flat=True)
         )
 
         # Calculate creates and updates
-        for path, exc in self.metadata[FIS].items():
+        for path, exc in failed_imports.items():
             if path in existing_failed_import_paths:
                 fis[_UPDATE_FIS][path] = exc
             else:
                 fis[_CREATE_FIS][path] = exc
 
         fis[_DELETE_FI_PATHS] |= self._query_failed_import_deletes(
-            existing_failed_import_paths
+            library, failed_imports, existing_failed_import_paths
         )
-        self.metadata.pop(FIS)
 
         count = 0
         for iterable in fis.values():
             count += len(iterable)
-        return
+        return count
 
-    def _bulk_update_failed_imports(self, update_failed_imports):
+    @status_notify()
+    def _bulk_update_failed_imports(
+        self, update_failed_imports, library, **kwargs
+    ) -> int:
         """Bulk update failed imports."""
         if not update_failed_imports:
-            return
+            return 0
         update_failed_import_objs = FailedImport.objects.filter(
-            library=self.library, path__in=update_failed_imports.keys()
+            library=library, path__in=update_failed_imports.keys()
         ).only(*_BULK_UPDATE_FAILED_IMPORT_FIELDS)
         if not update_failed_import_objs:
-            return
+            return 0
+        now = Now()
         for fi in update_failed_import_objs:
             try:
                 exc = update_failed_imports.pop(fi.path)
                 fi.set_reason(exc)
                 fi.presave()
-                fi.updated_at = Now()
+                fi.updated_at = now
             except Exception as exc:
                 self.log.exception(
                     f"Error preparing failed import update for {fi.path}"
@@ -101,16 +107,17 @@ class FailedImportsImporter(DeletedImporter):
         count = len(update_failed_import_objs)
         if count:
             self.log.info(f"Updated {count} old failed imports.")
-        return
+        return count
 
-    def _bulk_create_failed_imports(self, create_failed_imports):
+    @status_notify()
+    def _bulk_create_failed_imports(self, create_failed_imports, library, **kwargs):
         """Bulk create failed imports."""
         if not create_failed_imports:
             return 0
         create_objs = []
         for path, exc in create_failed_imports.items():
             try:
-                fi = FailedImport(library=self.library, path=path, parent_folder=None)
+                fi = FailedImport(library=library, path=path, parent_folder=None)
                 fi.set_reason(exc)
                 fi.presave()
                 create_objs.append(fi)
@@ -127,39 +134,54 @@ class FailedImportsImporter(DeletedImporter):
             self.log.info(f"Added {count} comics to failed imports.")
         return count
 
-    def _bulk_cleanup_failed_imports(self, delete_failed_imports_paths):
+    @status_notify()
+    def _bulk_cleanup_failed_imports(
+        self, delete_failed_imports_paths, library, **kwargs
+    ):
         """Remove FailedImport objects that have since succeeded."""
         if not delete_failed_imports_paths:
             return 0
         # Cleanup FailedImports that were actually successful
-        qs = FailedImport.objects.filter(library=self.library).filter(
+        qs = FailedImport.objects.filter(library=library).filter(
             path__in=delete_failed_imports_paths
         )
         qs.delete()
         count = len(delete_failed_imports_paths)
         if count:
-            self.log.info(f"Cleaned up {count} failed imports from {self.library.path}")
+            self.log.info(f"Cleaned up {count} failed imports from {library.path}")
         return count
 
-    def fail_imports(self):
+    def fail_imports(self, library, failed_imports, is_files_deleted):
         """Handle failed imports."""
-        fis = self.metadata.get(FIS)
-        if not fis:
-            return False
-        status = Status(ImportStatusTypes.FAILED_IMPORTS)
-        self.status_controller.start(status)
         created_count = 0
-
         try:
             fis = {_UPDATE_FIS: {}, _CREATE_FIS: {}, _DELETE_FI_PATHS: set()}
-            if self.task.files_deleted:
+            if is_files_deleted:
                 # if any files were deleted. Run the failed import check
-                self.metadata[FIS]["files_deleted"] = True
-            self._query_failed_imports(fis)
-            self._bulk_update_failed_imports(fis[_UPDATE_FIS])
-            self._bulk_create_failed_imports(fis[_CREATE_FIS])
-            self._bulk_cleanup_failed_imports(fis[_DELETE_FI_PATHS])
+                failed_imports["files_deleted"] = True
+            status = Status(ImportStatusTypes.FAILED_IMPORTS, 0, len(failed_imports))
+            status.total = self._query_failed_imports(
+                failed_imports,
+                library,
+                fis,
+                status=status,
+            )
+
+            self._bulk_update_failed_imports(
+                fis[_UPDATE_FIS],
+                library,
+                status=status,
+            )
+
+            created_count = self._bulk_create_failed_imports(
+                fis[_CREATE_FIS],
+                library,
+                status=status,
+            )
+
+            self._bulk_cleanup_failed_imports(
+                fis[_DELETE_FI_PATHS], library, status=status
+            )
         except Exception:
             self.log.exception("Processing failed imports")
-        self.status_controller.finish(status)
         return bool(created_count)

codex/librarian/importer/importerd.py

@@ -1,31 +1,299 @@
 """Bulk import and move comics and folders."""
 
-from django.utils import timezone
+import logging
+from pathlib import Path
+from time import sleep, time
 
-from codex.librarian.importer.importer import ComicImporter
+from django.core.cache import cache
+from humanize import naturaldelta
+
+from codex.librarian.importer.aggregate_metadata import AggregateMetadataMixin
+from codex.librarian.importer.const import FIS, FKS, M2M_MDS, MDS
+from codex.librarian.importer.deleted import DeletedMixin
+from codex.librarian.importer.failed_imports import FailedImportsMixin
+from codex.librarian.importer.moved import MovedMixin
 from codex.librarian.importer.status import ImportStatusTypes
 from codex.librarian.importer.tasks import (
     AdoptOrphanFoldersTask,
     ImportDBDiffTask,
     LazyImportComicsTask,
-    UpdateGroupsTask,
 )
-from codex.librarian.notifier.tasks import LIBRARY_CHANGED_TASK
-from codex.models import Comic, Folder, Library
+from codex.librarian.importer.update_comics import UpdateComicsMixin
+from codex.librarian.notifier.tasks import FAILED_IMPORTS_TASK, LIBRARY_CHANGED_TASK
+from codex.librarian.search.status import SearchIndexStatusTypes
+from codex.librarian.search.tasks import SearchIndexAbortTask, SearchIndexUpdateTask
+from codex.librarian.tasks import DelayedTasks
+from codex.models import Comic, Library
 from codex.status import Status
-from codex.threads import QueuedThread
+
+_WRITE_WAIT_EXPIRY = 60
 
 
-class ComicImporterThread(QueuedThread):
+class ComicImporterThread(
+    AggregateMetadataMixin,
+    DeletedMixin,
+    UpdateComicsMixin,
+    FailedImportsMixin,
+    MovedMixin,
+):
     """A worker to handle all bulk database updates."""
 
-    def _create_importer(self, task):
-        return ComicImporter(task, self.log_queue, self.librarian_queue)
+    def _wait_for_filesystem_ops_to_finish(self, task: ImportDBDiffTask) -> bool:
+        """Watchdog sends events before filesystem events finish, so wait for them."""
+        started_checking = time()
+
+        # Don't wait for deletes to complete.
+        # Do wait for move, modified, create files before import.
+        all_modified_paths = (
+            frozenset(task.dirs_moved.values())
+            | frozenset(task.files_moved.values())
+            | task.dirs_modified
+            | task.files_modified
+            | task.files_created
+        )
+
+        old_total_size = -1
+        total_size = 0
+        wait_time = 2
+        while old_total_size != total_size:
+            if old_total_size > 0:
+                # second time around or more
+                sleep(wait_time)
+                wait_time = wait_time**2
+                self.log.debug(
+                    f"Waiting for files to copy before import: "
+                    f"{old_total_size} != {total_size}"
+                )
+            if time() - started_checking > _WRITE_WAIT_EXPIRY:
+                return True
+
+            old_total_size = total_size
+            total_size = 0
+            for path_str in all_modified_paths:
+                path = Path(path_str)
+                if path.exists():
+                    total_size += Path(path).stat().st_size
+        return False
+
+    def _log_task_construct_dirs_log(self, task):
+        """Construct dirs log line."""
+        dirs_log = []
+        if task.dirs_moved:
+            dirs_log += [f"{len(task.dirs_moved)} moved"]
+        if task.dirs_modified:
+            dirs_log += [f"{len(task.dirs_modified)} modified"]
+        if task.dirs_deleted:
+            dirs_log += [f"{len(task.dirs_deleted)} deleted"]
+        return dirs_log
+
+    def _log_task_construct_comics_log(self, task):
+        """Construct comcis log line."""
+        comics_log = []
+        if task.files_moved:
+            comics_log += [f"{len(task.files_moved)} moved"]
+        if task.files_modified:
+            comics_log += [f"{len(task.files_modified)} modified"]
+        if task.files_created:
+            comics_log += [f"{len(task.files_created)} created"]
+        if task.files_deleted:
+            comics_log += [f"{len(task.files_deleted)} deleted"]
+        return comics_log
+
+    def _log_task(self, path, task):
+        """Log the watchdog event task."""
+        if self.log.getEffectiveLevel() < logging.DEBUG:
+            return
+
+        self.log.debug(f"Updating library {path}...")
+        comics_log = self._log_task_construct_comics_log(task)
+        if comics_log:
+            log = "Comics: "
+            log += ", ".join(comics_log)
+            self.log.debug("  " + log)
 
-    def _import(self, task):
-        """Run an import task."""
-        importer = self._create_importer(task)
-        importer.apply()
+        dirs_log = self._log_task_construct_dirs_log(task)
+        if dirs_log:
+            log = "Folders: "
+            log += ", ".join(dirs_log)
+            self.log.debug("  " + log)
+
+    @staticmethod
+    def _init_if_modified_or_created(task, path, status_list):
+        """Initialize librarian statuses for modified or created ops."""
+        total_paths = len(task.files_modified) + len(task.files_created)
+        status_list += [
+            Status(ImportStatusTypes.AGGREGATE_TAGS, 0, total_paths, subtitle=path),
+            Status(ImportStatusTypes.QUERY_MISSING_FKS),
+            Status(ImportStatusTypes.CREATE_FKS),
+        ]
+        if task.files_modified:
+            status_list += [
+                Status(
+                    ImportStatusTypes.FILES_MODIFIED,
+                    None,
+                    len(task.files_modified),
+                )
+            ]
+        if task.files_created:
+            status_list += [
+                Status(ImportStatusTypes.FILES_CREATED, None, len(task.files_created))
+            ]
+        if task.files_modified or task.files_created:
+            status_list += [Status(ImportStatusTypes.LINK_M2M_FIELDS)]
+        return total_paths
+
+    def _init_librarian_status(self, task, path):
+        """Update the librarian status tasks."""
+        status_list = []
+        search_index_updates = 0
+        if task.dirs_moved:
+            status_list += [
+                Status(ImportStatusTypes.DIRS_MOVED, None, len(task.dirs_moved))
+            ]
+        if task.files_moved:
+            status_list += [
+                Status(ImportStatusTypes.FILES_MOVED, None, len(task.files_moved))
+            ]
+            search_index_updates += len(task.files_moved)
+        if task.files_modified:
+            status_list += [
+                Status(ImportStatusTypes.DIRS_MODIFIED, None, len(task.dirs_modified))
+            ]
+        if task.files_modified or task.files_created:
+            search_index_updates += self._init_if_modified_or_created(
+                task, path, status_list
+            )
+        if task.files_deleted:
+            status_list += [
+                Status(ImportStatusTypes.FILES_DELETED, None, len(task.files_deleted))
+            ]
+            search_index_updates += len(task.files_deleted)
+        status_list += [
+            Status(
+                SearchIndexStatusTypes.SEARCH_INDEX_UPDATE,
+                0,
+                search_index_updates,
+            ),
+            Status(SearchIndexStatusTypes.SEARCH_INDEX_REMOVE),
+        ]
+        self.status_controller.start_many(status_list)
+
+    def _init_apply(self, library, task):
+        """Initialize the library and status flags."""
+        library.update_in_progress = True
+        library.save()
+        too_long = self._wait_for_filesystem_ops_to_finish(task)
+        if too_long:
+            self.log.warning(
+                "Import apply waited for the filesystem to stop changing too long. "
+                "Try polling again once files have finished copying"
+                f" in library: {library.path}"
+            )
+            return
+        self._log_task(library.path, task)
+        self._init_librarian_status(task, library.path)
+
+    def _create_comic_relations(self, library, fks) -> int:
+        """Query all foreign keys to determine what needs creating, then create them."""
+        if not fks:
+            return 0
+        create_data = self.query_all_missing_fks(library.path, fks)
+        return self.create_all_fks(library, create_data)
+
+    def _finish_apply_status(self, library):
+        """Finish all librarian statuses."""
+        library.update_in_progress = False
+        library.save()
+        self.status_controller.finish_many(ImportStatusTypes.values)
+
+    def _finish_apply(self, changed, new_failed_imports, changed_args):
+        """Perform final tasks when the apply is done."""
+        if changed:
+            library, start_time, imported_count = changed_args
+            self.librarian_queue.put(LIBRARY_CHANGED_TASK)
+            elapsed_time = time() - start_time
+            elapsed = naturaldelta(elapsed_time)
+            log_txt = f"Updated library {library.path} in {elapsed}."
+            if imported_count:
+                cps = round(imported_count / elapsed_time, 1)
+                log_txt += (
+                    f" Imported {imported_count} comics at {cps} comics per second."
+                )
+            else:
+                log_txt += " No comics to import."
+            self.log.info(log_txt)
+
+            # Wait to start the search index update in case more updates are incoming.
+            until = time() + 3
+            delayed_search_task = DelayedTasks(until, (SearchIndexUpdateTask(False),))
+            self.librarian_queue.put(delayed_search_task)
+        else:
+            self.log.info("No updates neccissary.")
+        if new_failed_imports:
+            self.librarian_queue.put(FAILED_IMPORTS_TASK)
+
+    def _apply(self, task):
+        """Bulk import comics."""
+        start_time = time()
+        self.librarian_queue.put(SearchIndexAbortTask())
+        library = Library.objects.get(pk=task.library_id)
+        try:
+            self._init_apply(library, task)
+
+            changed: int = 0
+            changed += self.move_and_modify_dirs(library, task)
+
+            modified_paths = task.files_modified
+            created_paths = task.files_created
+            task.files_modified = task.files_created = None
+            mds = {}
+            m2m_mds = {}
+            fks = {}
+            fis = {}
+            all_metadata = {
+                MDS: mds,
+                M2M_MDS: m2m_mds,
+                FKS: fks,
+                FIS: fis,
+            }
+            self.get_aggregate_metadata(
+                modified_paths | created_paths,
+                library.path,
+                all_metadata,
+                task.force_import_metadata,
+            )
+            all_metadata = None
+            modified_paths -= fis.keys()
+            created_paths -= fis.keys()
+
+            changed += self._create_comic_relations(library, fks)
+            fks = None
+
+            imported_count = self.bulk_update_comics(
+                modified_paths,
+                library,
+                created_paths,
+                mds,
+            )
+            modified_paths = None
+            imported_count += self.bulk_create_comics(created_paths, library, mds)
+            created_paths = mds = None
+            self.bulk_query_and_link_comic_m2m_fields(m2m_mds)
+            m2m_mds = None
+            changed += imported_count
+
+            new_failed_imports = self.fail_imports(
+                library, fis, bool(task.files_deleted)
+            )
+
+            changed += self.delete(library, task)
+            if changed:
+                cache.clear()
+        finally:
+            self._finish_apply_status(library)
+
+        changed_args = (library, start_time, imported_count)
+        self._finish_apply(changed, new_failed_imports, changed_args)
 
     def _lazy_import_metadata(self, task):
         """Kick off an import task for just these books."""
@@ -38,76 +306,27 @@ class ComicImporterThread(QueuedThread):
             library_path_map[library_id].add(import_comic.path)
 
         for library_id, paths in library_path_map.items():
-            # An abridged import task.
             task = ImportDBDiffTask(
                 library_id=library_id,
+                dirs_moved={},
+                files_moved={},
+                dirs_modified=frozenset(),
                 files_modified=frozenset(paths),
+                files_created=frozenset(),
+                dirs_deleted=frozenset(),
+                files_deleted=frozenset(),
                 force_import_metadata=True,
             )
-            self._import(task)
-
-    def _adopt_orphan_folders_for_library(self, library):
-        """Adopt orphan folders for one library."""
-        orphan_folder_paths = (
-            Folder.objects.filter(library=library, parent_folder=None)
-            .exclude(path=library.path)
-            .values_list("path", flat=True)
-        )
-        if not orphan_folder_paths:
-            self.log.debug(f"No orphan folders in {library.path}")
-            return False
-
-        self.log.debug(
-            f"{len(orphan_folder_paths)} orphan folders found in {library.path}"
-        )
-
-        # Move in place
-        folders_moved = {path: path for path in orphan_folder_paths}
-
-        # An abridged import task.
-        task = ImportDBDiffTask(
-            library_id=library.pk,
-            dirs_moved=folders_moved,
-        )
-        importer = self._create_importer(task)
-        # Only run the moved task.
-        importer.bulk_folders_moved()
-        return True
-
-    def _adopt_orphan_folders(self):
-        """Find orphan folders and move them into their correct place."""
-        status = Status(ImportStatusTypes.ADOPT_FOLDERS)
-        moved_status = Status(ImportStatusTypes.DIRS_MOVED)
-        self.status_controller.start(status)
-        self.status_controller.start(moved_status)
-        libraries = Library.objects.filter(covers_only=False).only("path")
-        for library in libraries.iterator():
-            folders_left = True
-            while folders_left:
-                # Run until there are no orphan folders
-                folders_left = self._adopt_orphan_folders_for_library(library)
-
-        self.status_controller.finish_many((moved_status, status))
-
-    def _update_groups(self, task):
-        pks = Library.objects.filter(covers_only=False).values_list("pk", flat=True)
-        start_time = task.start_time if task.start_time else timezone.now()
-        for pk in pks:
-            task = ImportDBDiffTask(library_id=pk)
-            importer = self._create_importer(task)
-            importer.update_all_groups({}, start_time)
-        self.librarian_queue.put(LIBRARY_CHANGED_TASK)
+        self._apply(task)
 
     def process_item(self, item):
         """Run the updater."""
         task = item
         if isinstance(task, ImportDBDiffTask):
-            self._import(task)
+            self._apply(task)
         elif isinstance(task, LazyImportComicsTask):
             self._lazy_import_metadata(task)
         elif isinstance(task, AdoptOrphanFoldersTask):
-            self._adopt_orphan_folders()
-        elif isinstance(task, UpdateGroupsTask):
-            self._update_groups(task)
+            self.adopt_orphan_folders()
         else:
             self.log.warning(f"Bad task sent to library updater {task}")