codex 1.5.17__py3-none-any.whl → 1.6.0__py3-none-any.whl

codex/librarian/importer/link_comics.py

@@ -10,6 +10,7 @@ from codex.librarian.importer.const import (
     DICT_MODEL_REL_LINK_MAP,
     FOLDERS_FIELD,
     IMPRINT,
+    M2M_MDS,
     PARENT_FOLDER,
     PUBLISHER,
     SERIES,
@@ -17,7 +18,8 @@ from codex.librarian.importer.const import (
     STORY_ARCS_METADATA_KEY,
     VOLUME,
 )
-from codex.librarian.importer.status import ImportStatusTypes, status_notify
+from codex.librarian.importer.link_covers import LinkCoversImporter
+from codex.librarian.importer.status import ImportStatusTypes
 from codex.models import (
     Comic,
     Folder,
@@ -27,10 +29,10 @@ from codex.models import (
     Volume,
 )
 from codex.models.named import Identifier
-from codex.threads import QueuedThread
+from codex.status import Status
 
 
-class LinkComicsMixin(QueuedThread):
+class LinkComicsImporter(LinkCoversImporter):
     """Link comics methods."""
 
     @staticmethod
@@ -40,13 +42,12 @@ class LinkComicsMixin(QueuedThread):
         return md.get(field_name, group_class.DEFAULT_NAME)
 
     @classmethod
-    def get_comic_fk_links(cls, md, library, path):
+    def get_comic_fk_links(cls, md, path):
         """Get links for all foreign keys for creating and updating."""
         publisher_name = cls._get_group_name(Publisher, md)
         imprint_name = cls._get_group_name(Imprint, md)
         series_name = cls._get_group_name(Series, md)
         volume_name = cls._get_group_name(Volume, md)
-        md["library"] = library
         md[PUBLISHER] = Publisher.objects.get(name=publisher_name)
         md[IMPRINT] = Imprint.objects.get(
             publisher__name=publisher_name, name=imprint_name
@@ -122,13 +123,15 @@ class LinkComicsMixin(QueuedThread):
         result = frozenset(pks)
         all_m2m_links[key][comic_pk] = result
 
-    def _link_comic_m2m_fields(self, m2m_mds):
+    def _link_comic_m2m_fields(self):
         """Get the complete m2m field data to create."""
         all_m2m_links = {}
-        comic_paths = frozenset(m2m_mds.keys())
+        comic_paths = frozenset(self.metadata.get(M2M_MDS, {}).keys())
+        if not comic_paths:
+            return all_m2m_links
         comics = Comic.objects.filter(path__in=comic_paths).values_list("pk", "path")
         for comic_pk, comic_path in comics:
-            md = m2m_mds[comic_path]
+            md = self.metadata[M2M_MDS][comic_path]
             link_data = (all_m2m_links, md, comic_pk)
             self._link_prepare_special_m2ms(
                 link_data, FOLDERS_FIELD, Folder, self._get_link_folders_filter
@@ -141,6 +144,7 @@ class LinkComicsMixin(QueuedThread):
                 self._link_prepare_special_m2ms(link_data, field_name, model, method)
 
             self._link_named_m2ms(all_m2m_links, comic_pk, md)
+        self.metadata.pop(M2M_MDS)
         return all_m2m_links
 
     def _query_relation_adjustments(
@@ -202,10 +206,9 @@ class LinkComicsMixin(QueuedThread):
         tms, all_del_pks = self._query_relation_adjustments(
             m2m_links, ThroughModel, through_field_id_name, status
         )
-        if status:
-            status.total = status.total or 0
-            status.total += len(tms) + len(all_del_pks)
-            self.status_controller.update(status)
+        status.total = status.total or 0
+        status.total += len(tms) + len(all_del_pks)
+        self.status_controller.update(status)
 
         update_fields = ("comic_id", through_field_id_name)
         ThroughModel.objects.bulk_create(
@@ -216,10 +219,9 @@ class LinkComicsMixin(QueuedThread):
         )
         if created_count := len(tms):
             self.log.info(f"Linked {created_count} new {field_name} to altered comics.")
-        if status:
-            status.complete = status.complete or 0
-            status.complete += created_count
-            self.status_controller.update(status)
+        status.complete = status.complete or 0
+        status.complete += created_count
+        self.status_controller.update(status)
 
         if del_count := len(all_del_pks):
             del_qs = ThroughModel.objects.filter(pk__in=all_del_pks)
@@ -227,19 +229,16 @@ class LinkComicsMixin(QueuedThread):
             self.log.info(
                 f"Deleted {del_count} stale {field_name} relations for altered comics.",
             )
-        if status:
-            status.complete = status.complete or 0
-            status.complete += del_count
-            self.status_controller.update(status)
+        status.complete = status.complete or 0
+        status.complete += del_count
+        self.status_controller.update(status)
         return created_count, del_count
 
-    @status_notify(status_type=ImportStatusTypes.LINK_M2M_FIELDS, updates=False)
-    def bulk_query_and_link_comic_m2m_fields(self, all_m2m_mds, status=None):
+    def bulk_query_and_link_comic_m2m_fields(self):
         """Combine query and bulk link into a batch."""
-        if status:
-            status.complete = 0
-            status.total = 0
-        all_m2m_links = self._link_comic_m2m_fields(all_m2m_mds)
+        status = Status(ImportStatusTypes.LINK_M2M_FIELDS)
+        self.status_controller.start(status)
+        all_m2m_links = self._link_comic_m2m_fields()
         created_total = 0
         del_total = 0
         for field_name, m2m_links in all_m2m_links.items():
@@ -256,4 +255,5 @@ class LinkComicsMixin(QueuedThread):
             self.log.info(f"Linked {created_total} comics to tags.")
         if del_total:
             self.log.info(f"Deleted {del_total} stale relations for altered comics.")
+        self.status_controller.finish(status)
         return created_total + del_total

codex/librarian/importer/link_covers.py

@@ -0,0 +1,73 @@
+"""Link Covers."""
+
+from pathlib import Path
+
+from codex.librarian.importer.const import CLASS_CUSTOM_COVER_GROUP_MAP, LINK_COVER_PKS
+from codex.librarian.importer.failed_imports import FailedImportsImporter
+from codex.librarian.importer.status import ImportStatusTypes
+from codex.models import CustomCover, Folder
+from codex.status import Status
+
+
+class LinkCoversImporter(FailedImportsImporter):
+    """Link Covers methods."""
+
+    def _link_custom_cover_prepare(self, cover, model_map):
+        """Prepare one cover in the model map for bulk update."""
+        if cover.library and cover.library.covers_only:
+            model = CLASS_CUSTOM_COVER_GROUP_MAP.inverse.get(cover.group)
+            if not model:
+                self.log.warning(f"Custom Cover model not found for {cover.path}")
+                return
+            group_filter = {"sort_name": cover.sort_name}
+        else:
+            model = Folder
+            path = str(Path(cover.path).parent)
+            group_filter = {"path": path}
+        qs = model.objects.filter(**group_filter).exclude(custom_cover=cover)
+        if not qs.exists():
+            return
+
+        if model not in model_map:
+            model_map[model] = []
+
+        for obj in qs.iterator():
+            obj.custom_cover = cover
+            model_map[model].append(obj)
+
+    def _link_custom_cover_group(self, model, objs, status):
+        """Bulk link a group to it's custom covers."""
+        count = 0
+        if not objs:
+            return count
+        model.objects.bulk_update(objs, ["custom_cover"])
+        count += len(objs)
+        self.log.info(f"Linked {count} custom covers to {model.__name__}s")
+        status.complete += count
+        self.status_controller.update(status, notify=False)
+        return count
+
+    def link_custom_covers(self):
+        """Link Custom Covers to Groups."""
+        link_cover_pks = self.metadata.get(LINK_COVER_PKS, {})
+        num_link_cover_pks = len(link_cover_pks)
+        if not num_link_cover_pks:
+            return 0
+        status = Status(ImportStatusTypes.COVERS_LINK, 0, num_link_cover_pks)
+        try:
+            self.status_controller.start(status)
+            # Aggregate objs to update for each group model.
+            model_map = {}
+            covers = CustomCover.objects.filter(pk__in=link_cover_pks).only(
+                "library", "path"
+            )
+            for cover in covers:
+                self._link_custom_cover_prepare(cover, model_map)
+
+            # Bulk update each model type
+            total_count = 0
+            for model, objs in model_map.items():
+                total_count += self._link_custom_cover_group(model, objs, status)
+        finally:
+            self.status_controller.finish(status)
+        return total_count

codex/librarian/importer/moved.py

@@ -4,74 +4,143 @@ from pathlib import Path
 
 from django.db.models.functions import Now
 
-from codex.librarian.importer.const import FOLDERS_FIELD, PARENT_FOLDER
-from codex.librarian.importer.create_comics import CreateComicsMixin
-from codex.librarian.importer.create_fks import (
+from codex.librarian.importer.aggregate import AggregateMetadataImporter
+from codex.librarian.importer.const import (
     BULK_UPDATE_FOLDER_MODIFIED_FIELDS,
-    CreateForeignKeysMixin,
+    CLASS_CUSTOM_COVER_GROUP_MAP,
+    CUSTOM_COVER_UPDATE_FIELDS,
+    FOLDERS_FIELD,
+    LINK_COVER_PKS,
+    MOVED_BULK_COMIC_UPDATE_FIELDS,
+    MOVED_BULK_FOLDER_UPDATE_FIELDS,
+    PARENT_FOLDER,
 )
-from codex.librarian.importer.query_fks import QueryForeignKeysMixin
-from codex.librarian.importer.status import ImportStatusTypes, status_notify
-from codex.models import Comic, Folder, Library
+from codex.librarian.importer.status import ImportStatusTypes
+from codex.models import Comic, CustomCover, Folder
+from codex.status import Status
 
-_MOVED_BULK_COMIC_UPDATE_FIELDS = ("path", "parent_folder")
-_MOVED_BULK_FOLDER_UPDATE_FIELDS = ("path", "parent_folder", "name")
 
-
-class MovedMixin(CreateComicsMixin, CreateForeignKeysMixin, QueryForeignKeysMixin):
+class MovedImporter(AggregateMetadataImporter):
     """Methods for moving comics and folders."""
 
-    @status_notify(status_type=ImportStatusTypes.FILES_MOVED, updates=False)
-    def _bulk_comics_moved(self, moved_paths, library, status=None):
+    def _bulk_comics_moved(self):
         """Move comcis."""
-        count = 0
-        if not moved_paths:
-            return count
+        num_files_moved = len(self.task.files_moved)
+        if not num_files_moved:
+            return
+        status = Status(ImportStatusTypes.FILES_MOVED, 0, num_files_moved)
+        self.status_controller.start(status)
 
         # Prepare FKs
         create_folder_paths = set()
-        self.query_missing_folder_paths(
-            moved_paths.values(), library.path, create_folder_paths, status=status
-        )
-        self.bulk_folders_create(create_folder_paths, library, status=status)
+        self.query_missing_folder_paths(create_folder_paths, status)
+        self.bulk_folders_create(frozenset(create_folder_paths), status)
 
         # Update Comics
         comics = Comic.objects.filter(
-            library=library, path__in=moved_paths.keys()
+            library=self.library, path__in=self.task.files_moved.keys()
         ).only("pk", "path", PARENT_FOLDER, FOLDERS_FIELD)
 
         folder_m2m_links = {}
-        now = Now()
         updated_comics = []
         for comic in comics.iterator():
             try:
-                new_path = moved_paths[comic.path]
+                new_path = self.task.files_moved[comic.path]
                 comic.path = new_path
                 new_path = Path(new_path)
                 comic.parent_folder = Folder.objects.get(  # type: ignore
                     path=new_path.parent
                 )
-                comic.updated_at = now
-                comic.set_stat()
+                comic.updated_at = Now()
+                comic.presave()
                 folder_m2m_links[comic.pk] = Folder.objects.filter(
                     path__in=new_path.parents
                 ).values_list("pk", flat=True)
                 updated_comics.append(comic)
             except Exception:
                 self.log.exception(f"moving {comic.path}")
+        self.task.files_moved = {}
 
-        Comic.objects.bulk_update(updated_comics, _MOVED_BULK_COMIC_UPDATE_FIELDS)
+        Comic.objects.bulk_update(updated_comics, MOVED_BULK_COMIC_UPDATE_FIELDS)
 
         # Update m2m field
         if folder_m2m_links:
-            self.bulk_fix_comic_m2m_field(FOLDERS_FIELD, folder_m2m_links, None)
+            self.bulk_fix_comic_m2m_field(FOLDERS_FIELD, folder_m2m_links, status)
         count = len(comics)
         if count:
             self.log.info(f"Moved {count} comics.")
 
-        return count
+        self.changed += count
+        self.status_controller.finish(status)
+
+    def _bulk_covers_moved_prepare(self, status):
+        """Create an update map for bulk update."""
+        covers = CustomCover.objects.filter(
+            library=self.library, path__in=self.task.covers_moved.keys()
+        ).only("pk", "path")
+
+        if status:
+            status.total = covers.count()
+
+        moved_covers = []
+        unlink_pks = set()
+        for cover in covers.iterator():
+            try:
+                new_path = self.task.covers_moved[cover.path]
+                cover.path = new_path
+                new_path = Path(new_path)
+                cover.updated_at = Now()
+                cover.presave()
+                moved_covers.append(cover)
+                unlink_pks.add(cover.pk)
+            except Exception:
+                self.log.exception(f"moving {cover.path}")
+        return moved_covers, unlink_pks
+
+    def _bulk_covers_moved_unlink(self, unlink_pks):
+        """Unlink moved covers because they could have moved between group dirs."""
+        if not unlink_pks:
+            return
+        self.log.debug(f"Unlinking {len(unlink_pks)} moved custom covers.")
+        for model in CLASS_CUSTOM_COVER_GROUP_MAP:
+            groups = model.objects.filter(custom_cover__in=unlink_pks)
+            unlink_groups = []
+            for group in groups:
+                group.custom_cover = None
+                unlink_groups.append(group)
+            if unlink_groups:
+                model.objects.bulk_update(unlink_groups, ["custom_cover"])
+                self.log.debug(
+                    f"Unlinked {len(unlink_groups)} {model.__name__} moved custom covers."
+                )
+
+        self._remove_covers(unlink_pks, custom=True)  # type: ignore
+
+    def _bulk_covers_moved(self, status=None):
+        """Move covers."""
+        num_covers_moved = len(self.task.covers_moved)
+        if not num_covers_moved:
+            return
+        status = Status(ImportStatusTypes.COVERS_MOVED, None, num_covers_moved)
+        self.status_controller.start(status)
+
+        moved_covers, unlink_pks = self._bulk_covers_moved_prepare(status)
+        if LINK_COVER_PKS not in self.metadata:
+            self.metadata[LINK_COVER_PKS] = set()
+        self.metadata[LINK_COVER_PKS].update(unlink_pks)
+        if moved_covers:
+            CustomCover.objects.bulk_update(moved_covers, CUSTOM_COVER_UPDATE_FIELDS)
+
+        self._bulk_covers_moved_unlink(unlink_pks)
 
-    def _get_parent_folders(self, library, dest_folder_paths, status):
+        count = len(moved_covers)
+        if count:
+            self.log.info(f"Moved {count} custom covers.")
+
+        self.changed += count
+        self.status_controller.finish(status)
+
+    def _get_parent_folders(self, dest_folder_paths, status):
         """Get destination parent folders."""
         # Determine parent folder paths.
         dest_parent_folder_paths = set()
@@ -82,12 +151,12 @@ class MovedMixin(CreateComicsMixin, CreateForeignKeysMixin, QueryForeignKeysMixi
 
         # Create only intermediate subfolders.
         existing_folder_paths = Folder.objects.filter(
-            library=library, path__in=dest_parent_folder_paths
+            library=self.library, path__in=dest_parent_folder_paths
         ).values_list("path", flat=True)
         create_folder_paths = frozenset(
             dest_parent_folder_paths - frozenset(existing_folder_paths)
         )
-        self.bulk_folders_create(create_folder_paths, library, status=status)
+        self.bulk_folders_create(create_folder_paths, status)
 
         # get parent folders path to model obj dict
         dest_parent_folders_objs = Folder.objects.filter(
@@ -98,93 +167,73 @@ class MovedMixin(CreateComicsMixin, CreateForeignKeysMixin, QueryForeignKeysMixi
             dest_parent_folders[folder.path] = folder
         return dest_parent_folders
 
-    @status_notify(status_type=ImportStatusTypes.DIRS_MOVED, updates=False)
-    def _bulk_folders_moved(self, folders_moved, library, **kwargs):
+    def bulk_folders_moved(self):
         """Move folders in the database instead of recreating them."""
-        if not folders_moved:
-            return 0
+        num_dirs_moved = len(self.task.dirs_moved)
+        if not num_dirs_moved:
+            return
+        status = Status(ImportStatusTypes.DIRS_MOVED, None, num_dirs_moved)
+        self.status_controller.start(status)
 
-        dest_folder_paths = frozenset(folders_moved.values())
-        status = kwargs.get("status")
-        dest_parent_folders = self._get_parent_folders(
-            library, dest_folder_paths, status
-        )
+        dest_folder_paths = frozenset(self.task.dirs_moved.values())
+        dest_parent_folders = self._get_parent_folders(dest_folder_paths, status)
 
-        src_folder_paths = frozenset(folders_moved.keys())
+        src_folder_paths = frozenset(self.task.dirs_moved.keys())
         folders_to_move = Folder.objects.filter(
-            library=library, path__in=src_folder_paths
+            library=self.library, path__in=src_folder_paths
         ).order_by("path")
 
         update_folders = []
-        now = Now()
         for folder in folders_to_move.iterator():
-            new_path = folders_moved[folder.path]
+            new_path = self.task.dirs_moved[folder.path]
             folder.name = Path(new_path).name
             folder.path = new_path
             parent_path_str = str(Path(new_path).parent)
             folder.parent_folder = dest_parent_folders.get(parent_path_str)
-            folder.set_stat()
-            folder.updated_at = now  # type: ignore
+            folder.presave()
+            folder.updated_at = Now()
             update_folders.append(folder)
+        self.task.dirs_moved = {}
 
         update_folders = sorted(update_folders, key=lambda x: len(Path(x.path).parts))
 
-        Folder.objects.bulk_update(update_folders, _MOVED_BULK_FOLDER_UPDATE_FIELDS)
+        Folder.objects.bulk_update(update_folders, MOVED_BULK_FOLDER_UPDATE_FIELDS)
         count = len(update_folders)
         if count:
             self.log.info(f"Moved {count} folders.")
-        return count
+        self.changed += count
+        self.status_controller.finish(status)
 
-    @status_notify(status_type=ImportStatusTypes.DIRS_MODIFIED, updates=False)
-    def bulk_folders_modified(self, paths, library, **kwargs):
+    def _bulk_folders_modified(self):
         """Update folders stat and nothing else."""
-        count = 0
-        if not paths:
-            return count
-        folders = Folder.objects.filter(library=library, path__in=paths).only(
-            "stat", "updated_at"
-        )
+        num_dirs_modified = len(self.task.dirs_modified)
+        if not num_dirs_modified:
+            return
+        status = Status(ImportStatusTypes.DIRS_MODIFIED, None, num_dirs_modified)
+        self.status_controller.start(status)
+
+        folders = Folder.objects.filter(
+            library=self.library, path__in=self.task.dirs_modified
+        ).only("stat", "updated_at")
+        self.task.dirs_modified = frozenset()
         update_folders = []
-        now = Now()
         for folder in folders.iterator():
             if Path(folder.path).exists():
-                folder.set_stat()
-                folder.updated_at = now
+                folder.updated_at = Now()
+                folder.presave()
                 update_folders.append(folder)
         Folder.objects.bulk_update(
             update_folders, fields=BULK_UPDATE_FOLDER_MODIFIED_FIELDS
         )
-        count += len(update_folders)
+        count = len(update_folders)
         if count:
             self.log.info(f"Modified {count} folders")
-        return count
+        self.changed += count
+        self.status_controller.finish(status)
 
-    def adopt_orphan_folders(self):
-        """Find orphan folders and move them into their correct place."""
-        libraries = Library.objects.only("pk", "path")
-        for library in libraries.iterator():
-            orphan_folder_paths = (
-                Folder.objects.filter(library=library, parent_folder=None)
-                .exclude(path=library.path)
-                .values_list("path", flat=True)
-            )
-
-            folders_moved = {}
-            for path in orphan_folder_paths:
-                folders_moved[path] = path
-
-            self._bulk_folders_moved(folders_moved, library)
-
-    def move_and_modify_dirs(self, library, task):
+    def move_and_modify_dirs(self):
         """Move files and dirs and modify dirs."""
-        changed = 0
-        changed += self._bulk_folders_moved(task.dirs_moved, library)
-        task.dirs_moved = None
-
-        changed += self._bulk_comics_moved(task.files_moved, library)
-        task.files_moved = None
-
-        changed += self.bulk_folders_modified(task.dirs_modified, library)
-        task.dirs_modified = None
-
-        return changed
+        self.bulk_folders_moved()
+        self._bulk_comics_moved()
+        self._bulk_covers_moved()
+        self._bulk_folders_modified()

codex/librarian/importer/query_covers.py

@@ -0,0 +1,52 @@
+"""Query Missing Custom Covers."""
+
+from codex.librarian.importer.const import (
+    COVERS_CREATE,
+    COVERS_UPDATE,
+)
+from codex.librarian.importer.create_fks import CreateForeignKeysImporter
+from codex.librarian.importer.status import ImportStatusTypes
+from codex.logger.logging import get_logger
+from codex.models.paths import CustomCover
+from codex.status import Status
+
+LOG = get_logger(__name__)
+
+
+class QueryCustomCoversImporter(CreateForeignKeysImporter):
+    """Query Missing Custom Covers."""
+
+    def query_missing_custom_covers(self):
+        """Identify update & create covers."""
+        cover_paths = self.task.covers_created | self.task.covers_modified
+        num_cover_paths = len(cover_paths)
+        if not num_cover_paths:
+            return
+        self.log.debug(f"Querying {num_cover_paths} custom cover_paths")
+        status = Status(ImportStatusTypes.QUERY_MISSING_COVERS, None, num_cover_paths)
+        self.status_controller.start(status)
+
+        update_covers_qs = CustomCover.objects.filter(
+            library=self.library, path__in=cover_paths
+        )
+        self.metadata[COVERS_UPDATE] = update_covers_qs
+        update_cover_paths = frozenset(update_covers_qs.values_list("path", flat=True))
+        update_count = len(update_cover_paths)
+        update_status = Status(ImportStatusTypes.COVERS_MODIFIED, 0, update_count)
+        self.status_controller.update(update_status, notify=False)
+
+        create_cover_paths = cover_paths - update_cover_paths
+        self.metadata[COVERS_CREATE] = create_cover_paths
+        create_count = len(create_cover_paths)
+        create_status = Status(ImportStatusTypes.COVERS_CREATED, 0, create_count)
+        self.status_controller.update(create_status, notify=False)
+
+        self.task.covers_created = self.task.covers_modified = frozenset()
+
+        count = create_count + update_count
+        if count:
+            self.log.debug(
+                f"Discovered {update_count} custom covers to update and {create_count} to create."
+            )
+        self.status_controller.finish(status)
+        return