codex 1.6.0a7__py3-none-any.whl → 1.6.0a9__py3-none-any.whl

codex/librarian/importer/moved.py

@@ -4,58 +4,54 @@ from pathlib import Path
 
 from django.db.models.functions import Now
 
+from codex.librarian.importer.aggregate import AggregateMetadataImporter
 from codex.librarian.importer.const import (
     BULK_UPDATE_FOLDER_MODIFIED_FIELDS,
     CLASS_CUSTOM_COVER_GROUP_MAP,
     CUSTOM_COVER_UPDATE_FIELDS,
     FOLDERS_FIELD,
+    LINK_COVER_PKS,
     MOVED_BULK_COMIC_UPDATE_FIELDS,
     MOVED_BULK_FOLDER_UPDATE_FIELDS,
     PARENT_FOLDER,
 )
-from codex.librarian.importer.create_comics import CreateComicsMixin
-from codex.librarian.importer.create_fks import (
-    CreateForeignKeysMixin,
-)
-from codex.librarian.importer.query_fks import QueryForeignKeysMixin
-from codex.librarian.importer.status import ImportStatusTypes, status_notify
-from codex.models import Comic, CustomCover, Folder, Library
+from codex.librarian.importer.status import ImportStatusTypes
+from codex.models import Comic, CustomCover, Folder
+from codex.status import Status
 
 
-class MovedMixin(CreateComicsMixin, CreateForeignKeysMixin, QueryForeignKeysMixin):
+class MovedImporter(AggregateMetadataImporter):
     """Methods for moving comics and folders."""
 
-    @status_notify(status_type=ImportStatusTypes.FILES_MOVED, updates=False)
-    def _bulk_comics_moved(self, moved_paths, library, status=None):
+    def _bulk_comics_moved(self):
         """Move comcis."""
-        count = 0
-        if not moved_paths:
-            return count
+        num_files_moved = len(self.task.files_moved)
+        if not num_files_moved:
+            return
+        status = Status(ImportStatusTypes.FILES_MOVED, 0, num_files_moved)
+        self.status_controller.start(status)
 
         # Prepare FKs
         create_folder_paths = set()
-        self.query_missing_folder_paths(
-            moved_paths.values(), library.path, create_folder_paths, status=status
-        )
-        self.bulk_folders_create(create_folder_paths, library, status=status)
+        self.query_missing_folder_paths(create_folder_paths, status)
+        self.bulk_folders_create(frozenset(create_folder_paths), status)
 
         # Update Comics
         comics = Comic.objects.filter(
-            library=library, path__in=moved_paths.keys()
+            library=self.library, path__in=self.task.files_moved.keys()
         ).only("pk", "path", PARENT_FOLDER, FOLDERS_FIELD)
 
         folder_m2m_links = {}
-        now = Now()
         updated_comics = []
         for comic in comics.iterator():
             try:
-                new_path = moved_paths[comic.path]
+                new_path = self.task.files_moved[comic.path]
                 comic.path = new_path
                 new_path = Path(new_path)
                 comic.parent_folder = Folder.objects.get(  # type: ignore
                     path=new_path.parent
                 )
-                comic.updated_at = now
+                comic.updated_at = Now()
                 comic.presave()
                 folder_m2m_links[comic.pk] = Folder.objects.filter(
                     path__in=new_path.parents
@@ -63,36 +59,37 @@ class MovedMixin(CreateComicsMixin, CreateForeignKeysMixin, QueryForeignKeysMixi
                 updated_comics.append(comic)
             except Exception:
                 self.log.exception(f"moving {comic.path}")
+        self.task.files_moved = {}
 
         Comic.objects.bulk_update(updated_comics, MOVED_BULK_COMIC_UPDATE_FIELDS)
 
         # Update m2m field
         if folder_m2m_links:
-            self.bulk_fix_comic_m2m_field(FOLDERS_FIELD, folder_m2m_links, None)
+            self.bulk_fix_comic_m2m_field(FOLDERS_FIELD, folder_m2m_links, status)
         count = len(comics)
         if count:
             self.log.info(f"Moved {count} comics.")
 
-        return count
+        self.changed += count
+        self.status_controller.finish(status)
 
-    def _bulk_covers_moved_prepare(self, library, moved_paths, status):
+    def _bulk_covers_moved_prepare(self, status):
         """Create an update map for bulk update."""
         covers = CustomCover.objects.filter(
-            library=library, path__in=moved_paths.keys()
+            library=self.library, path__in=self.task.covers_moved.keys()
         ).only("pk", "path")
 
         if status:
             status.total = covers.count()
 
-        now = Now()
         moved_covers = []
         unlink_pks = set()
         for cover in covers.iterator():
             try:
-                new_path = moved_paths[cover.path]
+                new_path = self.task.covers_moved[cover.path]
                 cover.path = new_path
                 new_path = Path(new_path)
-                cover.updated_at = now
+                cover.updated_at = Now()
                 cover.presave()
                 moved_covers.append(cover)
                 unlink_pks.add(cover.pk)
@@ -119,33 +116,31 @@ class MovedMixin(CreateComicsMixin, CreateForeignKeysMixin, QueryForeignKeysMixi
 
         self._remove_covers(unlink_pks, custom=True)  # type: ignore
 
-    @status_notify(status_type=ImportStatusTypes.FILES_MOVED, updates=False)
-    def _bulk_covers_moved(self, moved_paths, library, link_cover_pks, status=None):
+    def _bulk_covers_moved(self, status=None):
         """Move covers."""
-        count = 0
-        if not moved_paths:
-            return count
-        if status:
-            status.total = len(moved_paths)
+        num_covers_moved = len(self.task.covers_moved)
+        if not num_covers_moved:
+            return
+        status = Status(ImportStatusTypes.COVERS_MOVED, None, num_covers_moved)
+        self.status_controller.start(status)
 
-        moved_covers, unlink_pks = self._bulk_covers_moved_prepare(
-            library, moved_paths, status
-        )
-        link_cover_pks.update(unlink_pks)
+        moved_covers, unlink_pks = self._bulk_covers_moved_prepare(status)
+        if LINK_COVER_PKS not in self.metadata:
+            self.metadata[LINK_COVER_PKS] = set()
+        self.metadata[LINK_COVER_PKS].update(unlink_pks)
         if moved_covers:
             CustomCover.objects.bulk_update(moved_covers, CUSTOM_COVER_UPDATE_FIELDS)
-        count = len(moved_covers)
 
         self._bulk_covers_moved_unlink(unlink_pks)
 
+        count = len(moved_covers)
         if count:
             self.log.info(f"Moved {count} custom covers.")
-            if status:
-                status.add_complete(count)
 
-        return count
+        self.changed += count
+        self.status_controller.finish(status)
 
-    def _get_parent_folders(self, library, dest_folder_paths, status):
+    def _get_parent_folders(self, dest_folder_paths, status):
         """Get destination parent folders."""
         # Determine parent folder paths.
         dest_parent_folder_paths = set()
@@ -156,12 +151,12 @@ class MovedMixin(CreateComicsMixin, CreateForeignKeysMixin, QueryForeignKeysMixi
 
         # Create only intermediate subfolders.
         existing_folder_paths = Folder.objects.filter(
-            library=library, path__in=dest_parent_folder_paths
+            library=self.library, path__in=dest_parent_folder_paths
         ).values_list("path", flat=True)
         create_folder_paths = frozenset(
             dest_parent_folder_paths - frozenset(existing_folder_paths)
         )
-        self.bulk_folders_create(create_folder_paths, library, status=status)
+        self.bulk_folders_create(create_folder_paths, status)
 
         # get parent folders path to model obj dict
         dest_parent_folders_objs = Folder.objects.filter(
@@ -172,34 +167,33 @@ class MovedMixin(CreateComicsMixin, CreateForeignKeysMixin, QueryForeignKeysMixi
             dest_parent_folders[folder.path] = folder
         return dest_parent_folders
 
-    @status_notify(status_type=ImportStatusTypes.DIRS_MOVED, updates=False)
-    def _bulk_folders_moved(self, folders_moved, library, **kwargs):
+    def bulk_folders_moved(self):
         """Move folders in the database instead of recreating them."""
-        if not folders_moved:
-            return 0
+        num_dirs_moved = len(self.task.dirs_moved)
+        if not num_dirs_moved:
+            return
+        status = Status(ImportStatusTypes.DIRS_MOVED, None, num_dirs_moved)
+        self.status_controller.start(status)
 
-        dest_folder_paths = frozenset(folders_moved.values())
-        status = kwargs.get("status")
-        dest_parent_folders = self._get_parent_folders(
-            library, dest_folder_paths, status
-        )
+        dest_folder_paths = frozenset(self.task.dirs_moved.values())
+        dest_parent_folders = self._get_parent_folders(dest_folder_paths, status)
 
-        src_folder_paths = frozenset(folders_moved.keys())
+        src_folder_paths = frozenset(self.task.dirs_moved.keys())
         folders_to_move = Folder.objects.filter(
-            library=library, path__in=src_folder_paths
+            library=self.library, path__in=src_folder_paths
         ).order_by("path")
 
         update_folders = []
-        now = Now()
         for folder in folders_to_move.iterator():
-            new_path = folders_moved[folder.path]
+            new_path = self.task.dirs_moved[folder.path]
             folder.name = Path(new_path).name
             folder.path = new_path
             parent_path_str = str(Path(new_path).parent)
             folder.parent_folder = dest_parent_folders.get(parent_path_str)
             folder.presave()
-            folder.updated_at = now  # type: ignore
+            folder.updated_at = Now()
             update_folders.append(folder)
+        self.task.dirs_moved = {}
 
         update_folders = sorted(update_folders, key=lambda x: len(Path(x.path).parts))
 
@@ -207,61 +201,39 @@ class MovedMixin(CreateComicsMixin, CreateForeignKeysMixin, QueryForeignKeysMixi
         count = len(update_folders)
         if count:
             self.log.info(f"Moved {count} folders.")
-        return count
+        self.changed += count
+        self.status_controller.finish(status)
 
-    @status_notify(status_type=ImportStatusTypes.DIRS_MODIFIED, updates=False)
-    def bulk_folders_modified(self, paths, library, **kwargs):
+    def _bulk_folders_modified(self):
         """Update folders stat and nothing else."""
-        count = 0
-        if not paths:
-            return count
-        folders = Folder.objects.filter(library=library, path__in=paths).only(
-            "stat", "updated_at"
-        )
+        num_dirs_modified = len(self.task.dirs_modified)
+        if not num_dirs_modified:
+            return
+        status = Status(ImportStatusTypes.DIRS_MODIFIED, None, num_dirs_modified)
+        self.status_controller.start(status)
+
+        folders = Folder.objects.filter(
+            library=self.library, path__in=self.task.dirs_modified
+        ).only("stat", "updated_at")
+        self.task.dirs_modified = frozenset()
         update_folders = []
-        now = Now()
         for folder in folders.iterator():
             if Path(folder.path).exists():
-                folder.updated_at = now
+                folder.updated_at = Now()
                 folder.presave()
                 update_folders.append(folder)
         Folder.objects.bulk_update(
             update_folders, fields=BULK_UPDATE_FOLDER_MODIFIED_FIELDS
         )
-        count += len(update_folders)
+        count = len(update_folders)
         if count:
             self.log.info(f"Modified {count} folders")
-        return count
-
-    def adopt_orphan_folders(self):
-        """Find orphan folders and move them into their correct place."""
-        libraries = Library.objects.filter(covers_only=False).only("pk", "path")
-        for library in libraries.iterator():
-            orphan_folder_paths = (
-                Folder.objects.filter(library=library, parent_folder=None)
-                .exclude(path=library.path)
-                .values_list("path", flat=True)
-            )
-
-            folders_moved = {}
-            for path in orphan_folder_paths:
-                folders_moved[path] = path
+        self.changed += count
+        self.status_controller.finish(status)
 
-            self._bulk_folders_moved(folders_moved, library)
-
-    def move_and_modify_dirs(self, library, task, link_cover_pks):
+    def move_and_modify_dirs(self):
         """Move files and dirs and modify dirs."""
-        changed = 0
-        changed += self._bulk_folders_moved(task.dirs_moved, library)
-        task.dirs_moved = None
-
-        changed += self._bulk_comics_moved(task.files_moved, library)
-        task.files_moved = None
-
-        changed += self._bulk_covers_moved(task.covers_moved, library, link_cover_pks)
-        task.covers_moved = None
-
-        changed += self.bulk_folders_modified(task.dirs_modified, library)
-        task.dirs_modified = None
-
-        return changed
+        self.bulk_folders_moved()
+        self._bulk_comics_moved()
+        self._bulk_covers_moved()
+        self._bulk_folders_modified()

codex/librarian/importer/query_covers.py

@@ -0,0 +1,52 @@
+"""Query Missing Custom Covers."""
+
+from codex.librarian.importer.const import (
+    COVERS_CREATE,
+    COVERS_UPDATE,
+)
+from codex.librarian.importer.create_fks import CreateForeignKeysImporter
+from codex.librarian.importer.status import ImportStatusTypes
+from codex.logger.logging import get_logger
+from codex.models.paths import CustomCover
+from codex.status import Status
+
+LOG = get_logger(__name__)
+
+
+class QueryCustomCoversImporter(CreateForeignKeysImporter):
+    """Query Missing Custom Covers."""
+
+    def query_missing_custom_covers(self):
+        """Identify update & create covers."""
+        cover_paths = self.task.covers_created | self.task.covers_modified
+        num_cover_paths = len(cover_paths)
+        if not num_cover_paths:
+            return
+        self.log.debug(f"Querying {num_cover_paths} custom cover_paths")
+        status = Status(ImportStatusTypes.QUERY_MISSING_COVERS, None, num_cover_paths)
+        self.status_controller.start(status)
+
+        update_covers_qs = CustomCover.objects.filter(
+            library=self.library, path__in=cover_paths
+        )
+        self.metadata[COVERS_UPDATE] = update_covers_qs
+        update_cover_paths = frozenset(update_covers_qs.values_list("path", flat=True))
+        update_count = len(update_cover_paths)
+        update_status = Status(ImportStatusTypes.COVERS_MODIFIED, 0, update_count)
+        self.status_controller.update(update_status, notify=False)
+
+        create_cover_paths = cover_paths - update_cover_paths
+        self.metadata[COVERS_CREATE] = create_cover_paths
+        create_count = len(create_cover_paths)
+        create_status = Status(ImportStatusTypes.COVERS_CREATED, 0, create_count)
+        self.status_controller.update(create_status, notify=False)
+
+        self.task.covers_created = self.task.covers_modified = frozenset()
+
+        count = create_count + update_count
+        if count:
+            self.log.debug(
+                f"Discovered {update_count} custom covers to update and {create_count} to create."
+            )
+        self.status_controller.finish(status)
+        return