codex 1.6.0__py3-none-any.whl → 1.6.0a0__py3-none-any.whl

codex/librarian/importer/link_comics.py

@@ -10,7 +10,6 @@ from codex.librarian.importer.const import (
     DICT_MODEL_REL_LINK_MAP,
     FOLDERS_FIELD,
     IMPRINT,
-    M2M_MDS,
     PARENT_FOLDER,
     PUBLISHER,
     SERIES,
@@ -18,8 +17,7 @@ from codex.librarian.importer.const import (
     STORY_ARCS_METADATA_KEY,
     VOLUME,
 )
-from codex.librarian.importer.link_covers import LinkCoversImporter
-from codex.librarian.importer.status import ImportStatusTypes
+from codex.librarian.importer.status import ImportStatusTypes, status_notify
 from codex.models import (
     Comic,
     Folder,
@@ -29,10 +27,10 @@ from codex.models import (
     Volume,
 )
 from codex.models.named import Identifier
-from codex.status import Status
+from codex.threads import QueuedThread
 
 
-class LinkComicsImporter(LinkCoversImporter):
+class LinkComicsMixin(QueuedThread):
     """Link comics methods."""
 
     @staticmethod
@@ -42,12 +40,13 @@ class LinkComicsImporter(LinkCoversImporter):
         return md.get(field_name, group_class.DEFAULT_NAME)
 
     @classmethod
-    def get_comic_fk_links(cls, md, path):
+    def get_comic_fk_links(cls, md, library, path):
         """Get links for all foreign keys for creating and updating."""
         publisher_name = cls._get_group_name(Publisher, md)
         imprint_name = cls._get_group_name(Imprint, md)
         series_name = cls._get_group_name(Series, md)
         volume_name = cls._get_group_name(Volume, md)
+        md["library"] = library
         md[PUBLISHER] = Publisher.objects.get(name=publisher_name)
         md[IMPRINT] = Imprint.objects.get(
             publisher__name=publisher_name, name=imprint_name
@@ -123,15 +122,13 @@ class LinkComicsImporter(LinkCoversImporter):
         result = frozenset(pks)
         all_m2m_links[key][comic_pk] = result
 
-    def _link_comic_m2m_fields(self):
+    def _link_comic_m2m_fields(self, m2m_mds):
         """Get the complete m2m field data to create."""
         all_m2m_links = {}
-        comic_paths = frozenset(self.metadata.get(M2M_MDS, {}).keys())
-        if not comic_paths:
-            return all_m2m_links
+        comic_paths = frozenset(m2m_mds.keys())
         comics = Comic.objects.filter(path__in=comic_paths).values_list("pk", "path")
         for comic_pk, comic_path in comics:
-            md = self.metadata[M2M_MDS][comic_path]
+            md = m2m_mds[comic_path]
             link_data = (all_m2m_links, md, comic_pk)
             self._link_prepare_special_m2ms(
                 link_data, FOLDERS_FIELD, Folder, self._get_link_folders_filter
@@ -144,7 +141,6 @@ class LinkComicsImporter(LinkCoversImporter):
                 self._link_prepare_special_m2ms(link_data, field_name, model, method)
 
             self._link_named_m2ms(all_m2m_links, comic_pk, md)
-        self.metadata.pop(M2M_MDS)
         return all_m2m_links
 
     def _query_relation_adjustments(
@@ -206,9 +202,10 @@ class LinkComicsImporter(LinkCoversImporter):
         tms, all_del_pks = self._query_relation_adjustments(
             m2m_links, ThroughModel, through_field_id_name, status
         )
-        status.total = status.total or 0
-        status.total += len(tms) + len(all_del_pks)
-        self.status_controller.update(status)
+        if status:
+            status.total = status.total or 0
+            status.total += len(tms) + len(all_del_pks)
+            self.status_controller.update(status)
 
         update_fields = ("comic_id", through_field_id_name)
         ThroughModel.objects.bulk_create(
@@ -219,9 +216,10 @@ class LinkComicsImporter(LinkCoversImporter):
         )
         if created_count := len(tms):
             self.log.info(f"Linked {created_count} new {field_name} to altered comics.")
-        status.complete = status.complete or 0
-        status.complete += created_count
-        self.status_controller.update(status)
+        if status:
+            status.complete = status.complete or 0
+            status.complete += created_count
+            self.status_controller.update(status)
 
         if del_count := len(all_del_pks):
             del_qs = ThroughModel.objects.filter(pk__in=all_del_pks)
@@ -229,16 +227,19 @@ class LinkComicsImporter(LinkCoversImporter):
             self.log.info(
                 f"Deleted {del_count} stale {field_name} relations for altered comics.",
             )
-        status.complete = status.complete or 0
-        status.complete += del_count
-        self.status_controller.update(status)
+        if status:
+            status.complete = status.complete or 0
+            status.complete += del_count
+            self.status_controller.update(status)
         return created_count, del_count
 
-    def bulk_query_and_link_comic_m2m_fields(self):
+    @status_notify(status_type=ImportStatusTypes.LINK_M2M_FIELDS, updates=False)
+    def bulk_query_and_link_comic_m2m_fields(self, all_m2m_mds, status=None):
         """Combine query and bulk link into a batch."""
-        status = Status(ImportStatusTypes.LINK_M2M_FIELDS)
-        self.status_controller.start(status)
-        all_m2m_links = self._link_comic_m2m_fields()
+        if status:
+            status.complete = 0
+            status.total = 0
+        all_m2m_links = self._link_comic_m2m_fields(all_m2m_mds)
         created_total = 0
         del_total = 0
         for field_name, m2m_links in all_m2m_links.items():
@@ -255,5 +256,4 @@ class LinkComicsImporter(LinkCoversImporter):
             self.log.info(f"Linked {created_total} comics to tags.")
         if del_total:
             self.log.info(f"Deleted {del_total} stale relations for altered comics.")
-        self.status_controller.finish(status)
         return created_total + del_total

codex/librarian/importer/moved.py

@@ -4,54 +4,56 @@ from pathlib import Path
 
 from django.db.models.functions import Now
 
-from codex.librarian.importer.aggregate import AggregateMetadataImporter
 from codex.librarian.importer.const import (
     BULK_UPDATE_FOLDER_MODIFIED_FIELDS,
-    CLASS_CUSTOM_COVER_GROUP_MAP,
-    CUSTOM_COVER_UPDATE_FIELDS,
     FOLDERS_FIELD,
-    LINK_COVER_PKS,
     MOVED_BULK_COMIC_UPDATE_FIELDS,
     MOVED_BULK_FOLDER_UPDATE_FIELDS,
     PARENT_FOLDER,
 )
-from codex.librarian.importer.status import ImportStatusTypes
-from codex.models import Comic, CustomCover, Folder
-from codex.status import Status
+from codex.librarian.importer.create_comics import CreateComicsMixin
+from codex.librarian.importer.create_fks import (
+    CreateForeignKeysMixin,
+)
+from codex.librarian.importer.query_fks import QueryForeignKeysMixin
+from codex.librarian.importer.status import ImportStatusTypes, status_notify
+from codex.models import Comic, Folder, Library
 
 
-class MovedImporter(AggregateMetadataImporter):
+class MovedMixin(CreateComicsMixin, CreateForeignKeysMixin, QueryForeignKeysMixin):
     """Methods for moving comics and folders."""
 
-    def _bulk_comics_moved(self):
+    @status_notify(status_type=ImportStatusTypes.FILES_MOVED, updates=False)
+    def _bulk_comics_moved(self, moved_paths, library, status=None):
         """Move comcis."""
-        num_files_moved = len(self.task.files_moved)
-        if not num_files_moved:
-            return
-        status = Status(ImportStatusTypes.FILES_MOVED, 0, num_files_moved)
-        self.status_controller.start(status)
+        count = 0
+        if not moved_paths:
+            return count
 
         # Prepare FKs
         create_folder_paths = set()
-        self.query_missing_folder_paths(create_folder_paths, status)
-        self.bulk_folders_create(frozenset(create_folder_paths), status)
+        self.query_missing_folder_paths(
+            moved_paths.values(), library.path, create_folder_paths, status=status
+        )
+        self.bulk_folders_create(create_folder_paths, library, status=status)
 
         # Update Comics
         comics = Comic.objects.filter(
-            library=self.library, path__in=self.task.files_moved.keys()
+            library=library, path__in=moved_paths.keys()
         ).only("pk", "path", PARENT_FOLDER, FOLDERS_FIELD)
 
         folder_m2m_links = {}
+        now = Now()
         updated_comics = []
         for comic in comics.iterator():
             try:
-                new_path = self.task.files_moved[comic.path]
+                new_path = moved_paths[comic.path]
                 comic.path = new_path
                 new_path = Path(new_path)
                 comic.parent_folder = Folder.objects.get(  # type: ignore
                     path=new_path.parent
                 )
-                comic.updated_at = Now()
+                comic.updated_at = now
                 comic.presave()
                 folder_m2m_links[comic.pk] = Folder.objects.filter(
                     path__in=new_path.parents
@@ -59,88 +61,19 @@ class MovedImporter(AggregateMetadataImporter):
                 updated_comics.append(comic)
             except Exception:
                 self.log.exception(f"moving {comic.path}")
-        self.task.files_moved = {}
 
         Comic.objects.bulk_update(updated_comics, MOVED_BULK_COMIC_UPDATE_FIELDS)
 
         # Update m2m field
         if folder_m2m_links:
-            self.bulk_fix_comic_m2m_field(FOLDERS_FIELD, folder_m2m_links, status)
+            self.bulk_fix_comic_m2m_field(FOLDERS_FIELD, folder_m2m_links, None)
         count = len(comics)
         if count:
             self.log.info(f"Moved {count} comics.")
 
-        self.changed += count
-        self.status_controller.finish(status)
-
-    def _bulk_covers_moved_prepare(self, status):
-        """Create an update map for bulk update."""
-        covers = CustomCover.objects.filter(
-            library=self.library, path__in=self.task.covers_moved.keys()
-        ).only("pk", "path")
-
-        if status:
-            status.total = covers.count()
-
-        moved_covers = []
-        unlink_pks = set()
-        for cover in covers.iterator():
-            try:
-                new_path = self.task.covers_moved[cover.path]
-                cover.path = new_path
-                new_path = Path(new_path)
-                cover.updated_at = Now()
-                cover.presave()
-                moved_covers.append(cover)
-                unlink_pks.add(cover.pk)
-            except Exception:
-                self.log.exception(f"moving {cover.path}")
-        return moved_covers, unlink_pks
-
-    def _bulk_covers_moved_unlink(self, unlink_pks):
-        """Unlink moved covers because they could have moved between group dirs."""
-        if not unlink_pks:
-            return
-        self.log.debug(f"Unlinking {len(unlink_pks)} moved custom covers.")
-        for model in CLASS_CUSTOM_COVER_GROUP_MAP:
-            groups = model.objects.filter(custom_cover__in=unlink_pks)
-            unlink_groups = []
-            for group in groups:
-                group.custom_cover = None
-                unlink_groups.append(group)
-            if unlink_groups:
-                model.objects.bulk_update(unlink_groups, ["custom_cover"])
-                self.log.debug(
-                    f"Unlinked {len(unlink_groups)} {model.__name__} moved custom covers."
-                )
-
-        self._remove_covers(unlink_pks, custom=True)  # type: ignore
-
-    def _bulk_covers_moved(self, status=None):
-        """Move covers."""
-        num_covers_moved = len(self.task.covers_moved)
-        if not num_covers_moved:
-            return
-        status = Status(ImportStatusTypes.COVERS_MOVED, None, num_covers_moved)
-        self.status_controller.start(status)
-
-        moved_covers, unlink_pks = self._bulk_covers_moved_prepare(status)
-        if LINK_COVER_PKS not in self.metadata:
-            self.metadata[LINK_COVER_PKS] = set()
-        self.metadata[LINK_COVER_PKS].update(unlink_pks)
-        if moved_covers:
-            CustomCover.objects.bulk_update(moved_covers, CUSTOM_COVER_UPDATE_FIELDS)
-
-        self._bulk_covers_moved_unlink(unlink_pks)
+        return count
 
-        count = len(moved_covers)
-        if count:
-            self.log.info(f"Moved {count} custom covers.")
-
-        self.changed += count
-        self.status_controller.finish(status)
-
-    def _get_parent_folders(self, dest_folder_paths, status):
+    def _get_parent_folders(self, library, dest_folder_paths, status):
         """Get destination parent folders."""
         # Determine parent folder paths.
         dest_parent_folder_paths = set()
@@ -151,12 +84,12 @@ class MovedImporter(AggregateMetadataImporter):
 
         # Create only intermediate subfolders.
         existing_folder_paths = Folder.objects.filter(
-            library=self.library, path__in=dest_parent_folder_paths
+            library=library, path__in=dest_parent_folder_paths
         ).values_list("path", flat=True)
         create_folder_paths = frozenset(
             dest_parent_folder_paths - frozenset(existing_folder_paths)
         )
-        self.bulk_folders_create(create_folder_paths, status)
+        self.bulk_folders_create(create_folder_paths, library, status=status)
 
         # get parent folders path to model obj dict
         dest_parent_folders_objs = Folder.objects.filter(
@@ -167,33 +100,34 @@ class MovedImporter(AggregateMetadataImporter):
             dest_parent_folders[folder.path] = folder
         return dest_parent_folders
 
-    def bulk_folders_moved(self):
+    @status_notify(status_type=ImportStatusTypes.DIRS_MOVED, updates=False)
+    def _bulk_folders_moved(self, folders_moved, library, **kwargs):
         """Move folders in the database instead of recreating them."""
-        num_dirs_moved = len(self.task.dirs_moved)
-        if not num_dirs_moved:
-            return
-        status = Status(ImportStatusTypes.DIRS_MOVED, None, num_dirs_moved)
-        self.status_controller.start(status)
+        if not folders_moved:
+            return 0
 
-        dest_folder_paths = frozenset(self.task.dirs_moved.values())
-        dest_parent_folders = self._get_parent_folders(dest_folder_paths, status)
+        dest_folder_paths = frozenset(folders_moved.values())
+        status = kwargs.get("status")
+        dest_parent_folders = self._get_parent_folders(
+            library, dest_folder_paths, status
+        )
 
-        src_folder_paths = frozenset(self.task.dirs_moved.keys())
+        src_folder_paths = frozenset(folders_moved.keys())
         folders_to_move = Folder.objects.filter(
-            library=self.library, path__in=src_folder_paths
+            library=library, path__in=src_folder_paths
         ).order_by("path")
 
         update_folders = []
+        now = Now()
         for folder in folders_to_move.iterator():
-            new_path = self.task.dirs_moved[folder.path]
+            new_path = folders_moved[folder.path]
             folder.name = Path(new_path).name
             folder.path = new_path
             parent_path_str = str(Path(new_path).parent)
             folder.parent_folder = dest_parent_folders.get(parent_path_str)
             folder.presave()
-            folder.updated_at = Now()
+            folder.updated_at = now  # type: ignore
             update_folders.append(folder)
-        self.task.dirs_moved = {}
 
         update_folders = sorted(update_folders, key=lambda x: len(Path(x.path).parts))
 
@@ -201,39 +135,58 @@ class MovedImporter(AggregateMetadataImporter):
         count = len(update_folders)
         if count:
             self.log.info(f"Moved {count} folders.")
-        self.changed += count
-        self.status_controller.finish(status)
+        return count
 
-    def _bulk_folders_modified(self):
+    @status_notify(status_type=ImportStatusTypes.DIRS_MODIFIED, updates=False)
+    def bulk_folders_modified(self, paths, library, **kwargs):
         """Update folders stat and nothing else."""
-        num_dirs_modified = len(self.task.dirs_modified)
-        if not num_dirs_modified:
-            return
-        status = Status(ImportStatusTypes.DIRS_MODIFIED, None, num_dirs_modified)
-        self.status_controller.start(status)
-
-        folders = Folder.objects.filter(
-            library=self.library, path__in=self.task.dirs_modified
-        ).only("stat", "updated_at")
-        self.task.dirs_modified = frozenset()
+        count = 0
+        if not paths:
+            return count
+        folders = Folder.objects.filter(library=library, path__in=paths).only(
+            "stat", "updated_at"
+        )
         update_folders = []
+        now = Now()
         for folder in folders.iterator():
             if Path(folder.path).exists():
-                folder.updated_at = Now()
+                folder.updated_at = now
                 folder.presave()
                 update_folders.append(folder)
         Folder.objects.bulk_update(
             update_folders, fields=BULK_UPDATE_FOLDER_MODIFIED_FIELDS
         )
-        count = len(update_folders)
+        count += len(update_folders)
         if count:
             self.log.info(f"Modified {count} folders")
-        self.changed += count
-        self.status_controller.finish(status)
+        return count
 
-    def move_and_modify_dirs(self):
+    def adopt_orphan_folders(self):
+        """Find orphan folders and move them into their correct place."""
+        libraries = Library.objects.only("pk", "path")
+        for library in libraries.iterator():
+            orphan_folder_paths = (
+                Folder.objects.filter(library=library, parent_folder=None)
+                .exclude(path=library.path)
+                .values_list("path", flat=True)
+            )
+
+            folders_moved = {}
+            for path in orphan_folder_paths:
+                folders_moved[path] = path
+
+            self._bulk_folders_moved(folders_moved, library)
+
+    def move_and_modify_dirs(self, library, task):
         """Move files and dirs and modify dirs."""
-        self.bulk_folders_moved()
-        self._bulk_comics_moved()
-        self._bulk_covers_moved()
-        self._bulk_folders_modified()
+        changed = 0
+        changed += self._bulk_folders_moved(task.dirs_moved, library)
+        task.dirs_moved = None
+
+        changed += self._bulk_comics_moved(task.files_moved, library)
+        task.files_moved = None
+
+        changed += self.bulk_folders_modified(task.dirs_modified, library)
+        task.dirs_modified = None
+
+        return changed