codex 1.6.0a6__py3-none-any.whl → 1.6.0a8__py3-none-any.whl

codex/librarian/importer/importerd.py

@@ -1,400 +1,31 @@
 """Bulk import and move comics and folders."""
 
-import logging
-from pathlib import Path
-from time import sleep, time
+from django.utils.timezone import datetime
 
-from django.core.cache import cache
-from django.utils.timezone import now
-from humanize import naturaldelta
-
-from codex.librarian.importer.aggregate_metadata import AggregateMetadataMixin
-from codex.librarian.importer.const import FIS, FKS, M2M_MDS, MDS
-from codex.librarian.importer.covers import CoversMixin
-from codex.librarian.importer.deleted import DeletedMixin
-from codex.librarian.importer.failed_imports import FailedImportsMixin
-from codex.librarian.importer.moved import MovedMixin
+from codex.librarian.importer.importer import ComicImporter
 from codex.librarian.importer.status import ImportStatusTypes
 from codex.librarian.importer.tasks import (
     AdoptOrphanFoldersTask,
     ImportDBDiffTask,
     LazyImportComicsTask,
+    UpdateGroupsFirstComic,
 )
-from codex.librarian.importer.update_comics import UpdateComicsMixin
-from codex.librarian.notifier.tasks import FAILED_IMPORTS_TASK, LIBRARY_CHANGED_TASK
-from codex.librarian.search.status import SearchIndexStatusTypes
-from codex.librarian.search.tasks import SearchIndexAbortTask, SearchIndexUpdateTask
-from codex.librarian.tasks import DelayedTasks
-from codex.models import Comic, Library
+from codex.librarian.notifier.tasks import LIBRARY_CHANGED_TASK
+from codex.models import Comic, Folder, Library
 from codex.status import Status
-
-_WRITE_WAIT_EXPIRY = 60
+from codex.threads import QueuedThread
 
 
-class ComicImporterThread(
-    AggregateMetadataMixin,
-    DeletedMixin,
-    UpdateComicsMixin,
-    FailedImportsMixin,
-    MovedMixin,
-    CoversMixin,
-):
+class ComicImporterThread(QueuedThread):
     """A worker to handle all bulk database updates."""
 
-    def _wait_for_filesystem_ops_to_finish(self, task: ImportDBDiffTask) -> bool:
-        """Watchdog sends events before filesystem events finish, so wait for them."""
-        started_checking = time()
-
-        # Don't wait for deletes to complete.
-        # Do wait for move, modified, create files before import.
-        all_modified_paths = (
-            frozenset(task.dirs_moved.values())
-            | frozenset(task.files_moved.values())
-            | task.dirs_modified
-            | task.files_modified
-            | task.files_created
-        )
-
-        old_total_size = -1
-        total_size = 0
-        wait_time = 2
-        while old_total_size != total_size:
-            if old_total_size > 0:
-                # second time around or more
-                sleep(wait_time)
-                wait_time = wait_time**2
-                self.log.debug(
-                    f"Waiting for files to copy before import: "
-                    f"{old_total_size} != {total_size}"
-                )
-            if time() - started_checking > _WRITE_WAIT_EXPIRY:
-                return True
-
-            old_total_size = total_size
-            total_size = 0
-            for path_str in all_modified_paths:
-                path = Path(path_str)
-                if path.exists():
-                    total_size += Path(path).stat().st_size
-        return False
-
-    def _log_task_construct_dirs_log(self, task):
-        """Construct dirs log line."""
-        dirs_log = []
-        if task.dirs_moved:
-            dirs_log += [f"{len(task.dirs_moved)} moved"]
-        if task.dirs_modified:
-            dirs_log += [f"{len(task.dirs_modified)} modified"]
-        if task.dirs_deleted:
-            dirs_log += [f"{len(task.dirs_deleted)} deleted"]
-        return dirs_log
-
-    def _log_task_construct_comics_log(self, task):
-        """Construct comcis log line."""
-        comics_log = []
-        if task.files_moved:
-            comics_log += [f"{len(task.files_moved)} moved"]
-        if task.files_modified:
-            comics_log += [f"{len(task.files_modified)} modified"]
-        if task.files_created:
-            comics_log += [f"{len(task.files_created)} created"]
-        if task.files_deleted:
-            comics_log += [f"{len(task.files_deleted)} deleted"]
-        return comics_log
-
-    def _log_task(self, path, task):
-        """Log the watchdog event task."""
-        if self.log.getEffectiveLevel() < logging.DEBUG:
-            return
-
-        self.log.debug(f"Updating library {path}...")
-        comics_log = self._log_task_construct_comics_log(task)
-        if comics_log:
-            log = "Comics: "
-            log += ", ".join(comics_log)
-            self.log.debug("  " + log)
-
-        dirs_log = self._log_task_construct_dirs_log(task)
-        if dirs_log:
-            log = "Folders: "
-            log += ", ".join(dirs_log)
-            self.log.debug("  " + log)
-
-    @staticmethod
-    def _init_librarian_status_moved(task, status_list):
-        """Initialize moved statuses."""
-        search_index_updates = 0
-        if task.dirs_moved:
-            status_list += [
-                Status(ImportStatusTypes.DIRS_MOVED, None, len(task.dirs_moved))
-            ]
-        if task.files_moved:
-            status_list += [
-                Status(ImportStatusTypes.FILES_MOVED, None, len(task.files_moved))
-            ]
-            search_index_updates += len(task.files_moved)
-        if task.covers_moved:
-            status_list += [
-                Status(ImportStatusTypes.COVERS_MOVED, None, len(task.covers_moved))
-            ]
-        if task.dirs_modified:
-            status_list += [
-                Status(ImportStatusTypes.DIRS_MODIFIED, None, len(task.dirs_modified))
-            ]
-        return search_index_updates
-
-    @staticmethod
-    def _init_if_modified_or_created(task, path, status_list):
-        """Initialize librarian statuses for modified or created ops."""
-        total_paths = len(task.files_modified) + len(task.files_created)
-        status_list += [
-            Status(ImportStatusTypes.AGGREGATE_TAGS, 0, total_paths, subtitle=path),
-            Status(ImportStatusTypes.QUERY_MISSING_FKS),
-            Status(ImportStatusTypes.QUERY_MISSING_COVERS),
-            Status(ImportStatusTypes.CREATE_FKS),
-        ]
-        if task.files_modified:
-            status_list += [
-                Status(
-                    ImportStatusTypes.FILES_MODIFIED,
-                    None,
-                    len(task.files_modified),
-                )
-            ]
-        if task.covers_modified:
-            status_list += [
-                Status(
-                    ImportStatusTypes.COVERS_MODIFIED,
-                    None,
-                    len(task.covers_modified),
-                )
-            ]
-
-        if task.files_created or task.covers_created:
-            status_list += [
-                Status(ImportStatusTypes.FILES_CREATED, None, len(task.files_created))
-            ]
-        if task.covers_created:
-            status_list += [
-                Status(ImportStatusTypes.COVERS_CREATED, None, len(task.covers_created))
-            ]
-
-        if task.files_modified or task.files_created:
-            status_list += [Status(ImportStatusTypes.LINK_M2M_FIELDS)]
-
-        num_covers_linked = (
-            len(task.covers_moved)
-            + len(task.covers_modified)
-            + len(task.covers_created)
-        )
-        if num_covers_linked:
-            status_list += [
-                Status(ImportStatusTypes.COVERS_LINK, None, num_covers_linked)
-            ]
-        return total_paths
-
-    @staticmethod
-    def _init_librarian_status_deleted(task, status_list):
-        """Init deleted statuses."""
-        search_index_updates = 0
-        if task.files_deleted:
-            status_list += [
-                Status(ImportStatusTypes.FILES_DELETED, None, len(task.files_deleted))
-            ]
-            search_index_updates += len(task.files_deleted)
-        if task.covers_deleted:
-            status_list += [
-                Status(ImportStatusTypes.COVERS_DELETED, None, len(task.covers_deleted))
-            ]
-        return search_index_updates
-
-    @staticmethod
-    def _init_librarian_status_search_index(search_index_updates, status_list):
-        """Init search index statuses."""
-        status_list += [
-            Status(
-                SearchIndexStatusTypes.SEARCH_INDEX_UPDATE,
-                0,
-                search_index_updates,
-            ),
-            Status(SearchIndexStatusTypes.SEARCH_INDEX_REMOVE),
-        ]
-
-    def _init_librarian_status(self, task, path):
-        """Update the librarian status tasks."""
-        status_list = []
-        search_index_updates = 0
-        search_index_updates += self._init_librarian_status_moved(task, status_list)
-        if (
-            task.files_modified
-            or task.files_created
-            or task.covers_modified
-            or task.covers_created
-        ):
-            search_index_updates += self._init_if_modified_or_created(
-                task, path, status_list
-            )
-        search_index_updates += self._init_librarian_status_deleted(task, status_list)
-        self._init_librarian_status_search_index(search_index_updates, status_list)
-        self.status_controller.start_many(status_list)
-
-    def _init_apply(self, library, task):
-        """Initialize the library and status flags."""
-        library.update_in_progress = True
-        library.save()
-        too_long = self._wait_for_filesystem_ops_to_finish(task)
-        if too_long:
-            self.log.warning(
-                "Import apply waited for the filesystem to stop changing too long. "
-                "Try polling again once files have finished copying"
-                f" in library: {library.path}"
-            )
-            return
-        self._log_task(library.path, task)
-        self._init_librarian_status(task, library.path)
-
-    def _create_comic_and_cover_relations(
-        self, library, fks, cover_paths: frozenset[int], link_cover_pks: set[int]
-    ) -> int:
-        """Query all foreign keys to determine what needs creating, then create them."""
-        count = 0
-        if not fks and not cover_paths:
-            return count
-        create_data = self.query_all_missing_fks(library.path, fks)
-        query_cover_data = []
-        count += self.query_missing_custom_covers(
-            cover_paths,
-            library,
-            query_cover_data,
-        )
-        count += self.create_all_fks(library, create_data)
-        if query_cover_data:
-            update_covers_qs, create_cover_paths = query_cover_data
-            count += self.update_custom_covers(update_covers_qs, link_cover_pks)
-            link_covers_status = Status(
-                ImportStatusTypes.COVERS_LINK, 0, len(link_cover_pks)
-            )
-            self.status_controller.update(link_covers_status, notify=False)
-            count += self.create_custom_covers(
-                create_cover_paths,
-                library,
-                link_cover_pks,
-            )
-            link_covers_status = Status(
-                ImportStatusTypes.COVERS_LINK, 0, len(link_cover_pks)
-            )
-            self.status_controller.update(link_covers_status, notify=False)
-        return count
-
-    def _finish_apply_status(self, library):
-        """Finish all librarian statuses."""
-        library.update_in_progress = False
-        library.save()
-        self.status_controller.finish_many(ImportStatusTypes.values)
-
-    def _finish_apply(self, changed, new_failed_imports, changed_args):
-        """Perform final tasks when the apply is done."""
-        if changed:
-            library, start_time, imported_count = changed_args
-            self.librarian_queue.put(LIBRARY_CHANGED_TASK)
-            elapsed_time = time() - start_time
-            elapsed = naturaldelta(elapsed_time)
-            log_txt = f"Updated library {library.path} in {elapsed}."
-            if imported_count:
-                cps = round(imported_count / elapsed_time, 1)
-                log_txt += (
-                    f" Imported {imported_count} comics at {cps} comics per second."
-                )
-            else:
-                log_txt += " No comics to import."
-            self.log.info(log_txt)
-
-            # Wait to start the search index update in case more updates are incoming.
-            until = time() + 3
-            delayed_search_task = DelayedTasks(until, (SearchIndexUpdateTask(False),))
-            self.librarian_queue.put(delayed_search_task)
-        else:
-            self.log.info("No updates neccissary.")
-        if new_failed_imports:
-            self.librarian_queue.put(FAILED_IMPORTS_TASK)
-
-    def _apply(self, task):
-        """Bulk import comics."""
-        start_time = time()
-        self.librarian_queue.put(SearchIndexAbortTask())
-        library = Library.objects.get(pk=task.library_id)
-        try:
-            self._init_apply(library, task)
-            import_start_time = now()
-
-            changed: int = 0
-            link_cover_pks: set[int] = set()
-            changed += self.move_and_modify_dirs(library, task, link_cover_pks)
+    def _create_importer(self, task):
+        return ComicImporter(task, self.log_queue, self.librarian_queue)
 
-            modified_paths = task.files_modified
-            created_paths = task.files_created
-            task.files_modified = task.files_created = None
-
-            mds = {}
-            m2m_mds = {}
-            fks = {}
-            fis = {}
-            all_metadata = {
-                MDS: mds,
-                M2M_MDS: m2m_mds,
-                FKS: fks,
-                FIS: fis,
-            }
-            self.get_aggregate_metadata(
-                modified_paths | created_paths,
-                library.path,
-                all_metadata,
-                task.force_import_metadata,
-            )
-            all_metadata = None
-            modified_paths -= fis.keys()
-            created_paths -= fis.keys()
-
-            cover_paths = frozenset(task.covers_modified | task.covers_created)
-            task.covers_modified = task.covers_created = None
-
-            changed += self._create_comic_and_cover_relations(
-                library, fks, cover_paths, link_cover_pks
-            )
-            fks = cover_paths = None
-
-            imported_count = self.bulk_update_comics(
-                modified_paths,
-                library,
-                created_paths,
-                mds,
-            )
-            modified_paths = None
-            imported_count += self.bulk_create_comics(created_paths, library, mds)
-            created_paths = mds = None
-            self.bulk_query_and_link_comic_m2m_fields(m2m_mds)
-            m2m_mds = None
-            changed += imported_count
-
-            self.link_custom_covers(link_cover_pks)
-            link_cover_pks = set()
-
-            new_failed_imports = self.fail_imports(
-                library, fis, bool(task.files_deleted)
-            )
-
-            deleted_comic_groups = {}
-            changed += self.delete(library, task, deleted_comic_groups)
-
-            self.update_groups_for_covers(import_start_time, deleted_comic_groups)
-
-            if changed:
-                cache.clear()
-        finally:
-            self._finish_apply_status(library)
-
-        changed_args = (library, start_time, imported_count)
-        self._finish_apply(changed, new_failed_imports, changed_args)
+    def _import(self, task):
+        """Run an import task."""
+        importer = self._create_importer(task)
+        importer.apply()
 
     def _lazy_import_metadata(self, task):
         """Kick off an import task for just these books."""
@@ -407,31 +38,76 @@ class ComicImporterThread(
             library_path_map[library_id].add(import_comic.path)
 
         for library_id, paths in library_path_map.items():
+            # An abridged import task.
             task = ImportDBDiffTask(
                 library_id=library_id,
-                dirs_moved={},
-                files_moved={},
-                covers_moved={},
-                dirs_modified=frozenset(),
                 files_modified=frozenset(paths),
-                covers_modified=frozenset(),
-                files_created=frozenset(),
-                covers_created=frozenset(),
-                dirs_deleted=frozenset(),
-                files_deleted=frozenset(),
-                covers_deleted=frozenset(),
                 force_import_metadata=True,
             )
-        self._apply(task)
+            self._import(task)
+
+    def _adopt_orphan_folders_for_library(self, library):
+        """Adopt orphan folders for one library."""
+        orphan_folder_paths = (
+            Folder.objects.filter(library=library, parent_folder=None)
+            .exclude(path=library.path)
+            .values_list("path", flat=True)
+        )
+        if not orphan_folder_paths:
+            self.log.debug(f"No orphan folders in {library.path}")
+            return False
+
+        self.log.debug(
+            f"{len(orphan_folder_paths)} orphan folders found in {library.path}"
+        )
+
+        # Move in place
+        folders_moved = {path: path for path in orphan_folder_paths}
+
+        # An abridged import task.
+        task = ImportDBDiffTask(
+            library_id=library.pk,
+            dirs_moved=folders_moved,
+        )
+        importer = self._create_importer(task)
+        # Only run the moved task.
+        importer.bulk_folders_moved()
+        return True
+
+    def _adopt_orphan_folders(self):
+        """Find orphan folders and move them into their correct place."""
+        status = Status(ImportStatusTypes.ADOPT_FOLDERS)
+        moved_status = Status(ImportStatusTypes.DIRS_MOVED)
+        self.status_controller.start(status)
+        self.status_controller.start(moved_status)
+        libraries = Library.objects.filter(covers_only=False).only("path")
+        for library in libraries.iterator():
+            folders_left = True
+            while folders_left:
+                # Run until there are no orphan folders
+                folders_left = self._adopt_orphan_folders_for_library(library)
+
+        self.status_controller.finish_many((moved_status, status))
+
+    def _update_groups_first_comic(self, task):
+        pks = Library.objects.filter(covers_only=False).values_list("pk", flat=True)
+        start_time = task.start_time if task.start_time else datetime.now()
+        for pk in pks:
+            task = ImportDBDiffTask(library_id=pk)
+            importer = self._create_importer(task)
+            importer.update_all_groups_first_comics({}, start_time)
+        self.librarian_queue.put(LIBRARY_CHANGED_TASK)
 
     def process_item(self, item):
         """Run the updater."""
         task = item
         if isinstance(task, ImportDBDiffTask):
-            self._apply(task)
+            self._import(task)
         elif isinstance(task, LazyImportComicsTask):
             self._lazy_import_metadata(task)
         elif isinstance(task, AdoptOrphanFoldersTask):
-            self.adopt_orphan_folders()
+            self._adopt_orphan_folders()
+        elif isinstance(task, UpdateGroupsFirstComic):
+            self._update_groups_first_comic(task)
         else:
             self.log.warning(f"Bad task sent to library updater {task}")