codex 1.4.0a0__py3-none-any.whl → 1.4.1__py3-none-any.whl

codex/librarian/importer/create_fks.py

@@ -2,10 +2,9 @@
 
 So we may safely create the comics next.
 """
+from itertools import chain
 from pathlib import Path
 
-from django.db.models.functions import Now
-
 from codex.librarian.importer.status import ImportStatusTypes, status_notify
 from codex.models import (
     Creator,
@@ -15,11 +14,14 @@ from codex.models import (
     Imprint,
     Publisher,
     Series,
+    StoryArc,
+    StoryArcNumber,
     Volume,
 )
+from codex.status import Status
 from codex.threads import QueuedThread
 
-_BULK_UPDATE_FOLDER_MODIFIED_FIELDS = ("stat", "updated_at")
+BULK_UPDATE_FOLDER_MODIFIED_FIELDS = ("stat", "updated_at")
 _COUNT_FIELDS = {Series: "volume_count", Volume: "issue_count"}
 _GROUP_UPDATE_FIELDS = {
     Publisher: ("name",),
@@ -28,7 +30,11 @@ _GROUP_UPDATE_FIELDS = {
     Volume: ("name", "publisher", "imprint", "series"),
 }
 _NAMED_MODEL_UPDATE_FIELDS = ("name",)
-_CREATOR_UPDATE_FIELDS = ("person", "role")
+
+_CREATE_DICT_UPDATE_FIELDS = {
+    Creator: ("person", "role"),
+    StoryArcNumber: ("story_arc", "number"),
+}
 
 
 class CreateForeignKeysMixin(QueuedThread):
@@ -63,8 +69,8 @@ class CreateForeignKeysMixin(QueuedThread):
     @staticmethod
     def _update_group_obj(group_class, group_param_tuple, count_dict, count_field):
         """Update group counts for a Series or Volume."""
-        if count_dict is None:
-            # TODO this is never none now.
+        count = count_dict.get(count_field)
+        if count is None:
             return None
         search_kwargs = {
             "publisher__name": group_param_tuple[0],
@@ -76,15 +82,14 @@ class CreateForeignKeysMixin(QueuedThread):
 
         obj = group_class.objects.get(**search_kwargs)
         obj_count = getattr(obj, count_field)
-        count = count_dict.get(count_field)
-        if obj_count is None or (count is not None and obj_count < count):
+        if obj_count is None or count > obj_count:
             setattr(obj, count_field, count)
         else:
             obj = None
         return obj
 
     @status_notify()
-    def bulk_group_creator(self, group_tree_counts, group_class, status=None):
+    def _bulk_group_creator(self, group_tree_counts, group_class, status=None):
         """Bulk creates groups."""
         count = 0
         if not group_tree_counts:
@@ -103,12 +108,13 @@ class CreateForeignKeysMixin(QueuedThread):
         count += len(create_groups)
         self.log.info(f"Created {count} {group_class.__name__}s.")
         if status:
+            status.complete = status.complete or 0
             status.complete += count
             self.status_controller.update(status)
         return count
 
     @status_notify()
-    def bulk_group_updater(self, group_tree_counts, group_class, status=None):
+    def _bulk_group_updater(self, group_tree_counts, group_class, status=None):
         """Bulk update groups."""
         count = 0
         if not group_tree_counts:
@@ -125,33 +131,11 @@ class CreateForeignKeysMixin(QueuedThread):
         count += len(update_groups)
         self.log.info(f"Updated {count} {group_class.__name__}s.")
         if status:
+            status.complete = status.complete or 0
             status.complete += count
             self.status_controller.update(status)
         return count
 
-    @status_notify(status_type=ImportStatusTypes.DIRS_MODIFIED, updates=False)
-    def bulk_folders_modified(self, paths, library, **kwargs):
-        """Update folders stat and nothing else."""
-        count = 0
-        if not paths:
-            return count
-        folders = Folder.objects.filter(library=library, path__in=paths).only(
-            "stat", "updated_at"
-        )
-        update_folders = []
-        now = Now()
-        for folder in folders.iterator():
-            if Path(folder.path).exists():
-                folder.set_stat()
-                folder.updated_at = now
-                update_folders.append(folder)
-        Folder.objects.bulk_update(
-            update_folders, fields=_BULK_UPDATE_FOLDER_MODIFIED_FIELDS
-        )
-        count += len(update_folders)
-        self.log.info(f"Modified {count} folders")
-        return count
-
     @status_notify()
     def bulk_folders_create(self, folder_paths, library, status=None):
         """Create folders breadth first."""
@@ -176,9 +160,10 @@ class CreateForeignKeysMixin(QueuedThread):
                 try:
                     parent = Folder.objects.get(path=parent_path)
                 except Folder.DoesNotExist:
-                    if parent_path != library.path:
+                    if path.parent != Path(library.path):
                         self.log.exception(
-                            f"Can't find parent folder {parent_path} for {path}"
+                            f"Can't find parent folder {parent_path}"
+                            f" for {path} in library {library.path}"
                         )
                 folder = Folder(
                     library=library,
@@ -191,19 +176,20 @@ class CreateForeignKeysMixin(QueuedThread):
             Folder.objects.bulk_create(
                 create_folders,
                 update_conflicts=True,
-                update_fields=_BULK_UPDATE_FOLDER_MODIFIED_FIELDS,
+                update_fields=BULK_UPDATE_FOLDER_MODIFIED_FIELDS,
                 unique_fields=Folder._meta.unique_together[0],  # type: ignore
             )
             count += len(create_folders)
             if status:
-                status.complete = len(create_folders)
+                status.complete = status.complete or 0
+                status.complete += len(create_folders)
                 self.status_controller.update(status)
 
         self.log.info(f"Created {count} Folders.")
         return count
 
     @status_notify()
-    def bulk_create_named_models(self, names, named_class, status=None):
+    def _bulk_create_named_models(self, names, named_class, status=None):
         """Bulk create named models."""
         count = len(names)
         if not count:
@@ -221,33 +207,146 @@ class CreateForeignKeysMixin(QueuedThread):
         )
         self.log.info(f"Created {count} {named_class.__name__}s.")
         if status:
+            status.complete = status.complete or 0
             status.complete += count
             self.status_controller.update(status)
         return count
 
-    @status_notify()
-    def bulk_create_creators(self, create_creator_tuples, status=None):
-        """Bulk create creators."""
-        if not create_creator_tuples:
-            return 0
+    @staticmethod
+    def _create_creator(role_name, person_name):
+        role = CreatorRole.objects.get(name=role_name) if role_name else None
+        person = CreatorPerson.objects.get(name=person_name)
+        return Creator(role=role, person=person)
 
-        create_creators = []
-        for role_name, person_name in create_creator_tuples:
-            role = CreatorRole.objects.get(name=role_name) if role_name else None
-            person = CreatorPerson.objects.get(name=person_name)
-            creator = Creator(role=role, person=person)
+    @staticmethod
+    def _create_story_arc_number(name, number):
+        story_arc = StoryArc.objects.get(name=name)
+        return StoryArcNumber(story_arc=story_arc, number=number)
 
-            create_creators.append(creator)
+    def _bulk_create_dict_models(
+        self, create_tuples, create_method, model, status=None
+    ):
+        """Bulk create a dict type m2m model."""
+        if not create_tuples:
+            return 0
 
-        Creator.objects.bulk_create(
-            create_creators,
+        create_objs = []
+        for key, value in create_tuples:
+            obj = create_method(key, value)
+            create_objs.append(obj)
+
+        model.objects.bulk_create(
+            create_objs,
             update_conflicts=True,
-            update_fields=_CREATOR_UPDATE_FIELDS,
-            unique_fields=creator._meta.unique_together[0],  # type: ignore
+            update_fields=_CREATE_DICT_UPDATE_FIELDS[model],
+            unique_fields=model._meta.unique_together[0],  # type: ignore
         )
-        count = len(create_creators)
-        self.log.info(f"Created {count} creators.")
+        count = len(create_objs)
+        self.log.info(f"Created {count} {model.__class__.__name__}s.")
         if status:
+            status.complete = status.complete or 0
             status.complete += count
             self.status_controller.update(status)
         return count
+
+    @status_notify()
+    def _bulk_create_creators(self, create_creator_tuples, status=None):
+        """Bulk create creators."""
+        return self._bulk_create_dict_models(
+            create_creator_tuples,
+            self._create_creator,
+            Creator,
+            status,
+        )
+
+    @status_notify()
+    def _bulk_create_story_arc_numbers(
+        self, create_story_arc_number_tuples, status=None
+    ):
+        """Bulk create story_arc_numbers."""
+        return self._bulk_create_dict_models(
+            create_story_arc_number_tuples,
+            self._create_story_arc_number,
+            StoryArcNumber,
+            status,
+        )
+
+    @staticmethod
+    def _get_create_fks_totals(create_data):
+        (
+            create_groups,
+            update_groups,
+            create_folder_paths,
+            create_fks,
+            create_creators,
+            create_story_arc_numbers,
+        ) = create_data
+        total_fks = 0
+        for data_group in chain(
+            create_groups.values(), update_groups.values(), create_fks.values()
+        ):
+            total_fks += len(data_group)
+        total_fks += (
+            len(create_folder_paths)
+            + len(create_creators)
+            + len(create_story_arc_numbers)
+        )
+        return total_fks
+
+    def create_all_fks(self, library, create_data):
+        """Bulk create all foreign keys."""
+        total_fks = self._get_create_fks_totals(create_data)
+        status = Status(ImportStatusTypes.CREATE_FKS, 0, total_fks)
+        try:
+            self.status_controller.start(status)
+            (
+                create_groups,
+                update_groups,
+                create_folder_paths,
+                create_fks,
+                create_creators,
+                create_story_arc_numbers,
+            ) = create_data
+
+            for group_class, group_tree_counts in create_groups.items():
+                status.complete += self._bulk_group_creator(  # type: ignore
+                    group_tree_counts,
+                    group_class,
+                    status=status,
+                )
+
+            for group_class, group_tree_counts in update_groups.items():
+                status.complete += self._bulk_group_updater(  # type: ignore
+                    group_tree_counts,
+                    group_class,
+                    status=status,
+                )
+
+            status.complete += self.bulk_folders_create(  # type: ignore
+                sorted(create_folder_paths),
+                library,
+                status=status,
+            )
+
+            for named_class, names in create_fks.items():
+                status.complete += self._bulk_create_named_models(
+                    names,
+                    named_class,
+                    status=status,
+                )
+
+            # This must happen after creator_fks created by create_named_models
+            status.complete += self._bulk_create_creators(
+                create_creators,
+                status=status,
+            )
+
+            # This must happen after story_arc_fks created by create_named_models
+            status.complete += self._bulk_create_story_arc_numbers(
+                create_story_arc_numbers,
+                status=status,
+            )
+
+        finally:
+            self.status_controller.finish(status)
+        return status.complete

codex/librarian/importer/deleted.py

@@ -14,7 +14,7 @@ class DeletedMixin(QueuedThread):
         self.librarian_queue.put(task)
 
     @status_notify(status_type=ImportStatusTypes.DIRS_DELETED, updates=False)
-    def bulk_folders_deleted(self, delete_folder_paths, library, **kwargs):
+    def _bulk_folders_deleted(self, delete_folder_paths, library, **kwargs):
         """Bulk delete folders."""
         if not delete_folder_paths:
             return 0
@@ -36,7 +36,7 @@ class DeletedMixin(QueuedThread):
         return count
 
     @status_notify(status_type=ImportStatusTypes.FILES_DELETED, updates=False)
-    def bulk_comics_deleted(self, delete_comic_paths, library, **kwargs):
+    def _bulk_comics_deleted(self, delete_comic_paths, library, **kwargs):
         """Bulk delete comics found missing from the filesystem."""
         if not delete_comic_paths:
             return 0
@@ -50,3 +50,12 @@ class DeletedMixin(QueuedThread):
         self.log.info(f"Deleted {count} comics from {library.path}")
 
         return count
+
+    def delete(self, library, task):
+        """Delete files and folders."""
+        count = self._bulk_folders_deleted(task.dirs_deleted, library)
+        task.dirs_deleted = None
+
+        count += self._bulk_comics_deleted(task.files_deleted, library)
+        task.files_deleted = None
+        return count

codex/librarian/importer/failed_imports.py

@@ -3,8 +3,9 @@ from pathlib import Path
 
 from django.db.models.functions import Now
 
-from codex.librarian.importer.status import status_notify
+from codex.librarian.importer.status import ImportStatusTypes, status_notify
 from codex.models import Comic, FailedImport
+from codex.status import Status
 from codex.threads import QueuedThread
 
 _BULK_UPDATE_FAILED_IMPORT_FIELDS = ("name", "stat", "updated_at")
@@ -38,7 +39,7 @@ class FailedImportsMixin(QueuedThread):
         return succeeded_failed_imports | missing_failed_imports
 
     @status_notify()
-    def query_failed_imports(
+    def _query_failed_imports(
         self,
         failed_imports,
         library,
@@ -49,6 +50,9 @@ class FailedImportsMixin(QueuedThread):
         if not failed_imports:
             return 0
 
+        # Remove the files deleted hack thing.
+        failed_imports.pop("files_deleted", None)
+
         existing_failed_import_paths = set(
             FailedImport.objects.filter(library=library).values_list("path", flat=True)
         )
@@ -70,7 +74,7 @@ class FailedImportsMixin(QueuedThread):
         return count
 
     @status_notify()
-    def bulk_update_failed_imports(
+    def _bulk_update_failed_imports(
         self, update_failed_imports, library, **kwargs
     ) -> int:
         """Bulk update failed imports."""
@@ -102,7 +106,7 @@ class FailedImportsMixin(QueuedThread):
         return count
 
     @status_notify()
-    def bulk_create_failed_imports(self, create_failed_imports, library, **kwargs):
+    def _bulk_create_failed_imports(self, create_failed_imports, library, **kwargs):
         """Bulk create failed imports."""
         if not create_failed_imports:
             return 0
@@ -127,7 +131,7 @@ class FailedImportsMixin(QueuedThread):
         return count
 
     @status_notify()
-    def bulk_cleanup_failed_imports(
+    def _bulk_cleanup_failed_imports(
         self, delete_failed_imports_paths, library, **kwargs
     ):
         """Remove FailedImport objects that have since succeeded."""
@@ -141,3 +145,38 @@ class FailedImportsMixin(QueuedThread):
         count = len(delete_failed_imports_paths)
         self.log.info(f"Cleaned up {count} failed imports from {library.path}")
         return count
+
+    def fail_imports(self, library, failed_imports, is_files_deleted):
+        """Handle failed imports."""
+        created_count = 0
+        try:
+            fis = {"update_fis": {}, "create_fis": {}, "delete_fi_paths": set()}
+            if is_files_deleted:
+                # if any files were deleted. Run the failed import check
+                failed_imports["files_deleted"] = True
+            status = Status(ImportStatusTypes.FAILED_IMPORTS, 0, len(failed_imports))
+            status.total = self._query_failed_imports(
+                failed_imports,
+                library,
+                fis,
+                status=status,
+            )
+
+            self._bulk_update_failed_imports(
+                fis["update_fis"],
+                library,
+                status=status,
+            )
+
+            created_count = self._bulk_create_failed_imports(
+                fis["create_fis"],
+                library,
+                status=status,
+            )
+
+            self._bulk_cleanup_failed_imports(
+                fis["delete_fi_paths"], library, status=status
+            )
+        except Exception:
+            self.log.exception("Processing failed imports")
+        return bool(created_count)

codex/librarian/importer/importerd.py

@@ -6,9 +6,13 @@ from time import sleep, time
 from django.core.cache import cache
 from humanize import naturaldelta
 
-from codex.librarian.importer.db_ops import ApplyDBOpsMixin
+from codex.librarian.importer.aggregate_metadata import AggregateMetadataMixin
+from codex.librarian.importer.deleted import DeletedMixin
+from codex.librarian.importer.failed_imports import FailedImportsMixin
+from codex.librarian.importer.moved import MovedMixin
 from codex.librarian.importer.status import ImportStatusTypes
 from codex.librarian.importer.tasks import AdoptOrphanFoldersTask, UpdaterDBDiffTask
+from codex.librarian.importer.update_comics import UpdateComicsMixin
 from codex.librarian.notifier.tasks import FAILED_IMPORTS_TASK, LIBRARY_CHANGED_TASK
 from codex.librarian.search.status import SearchIndexStatusTypes
 from codex.librarian.search.tasks import SearchIndexAbortTask, SearchIndexUpdateTask
@@ -19,7 +23,13 @@ from codex.status import Status
 _WRITE_WAIT_EXPIRY = 60
 
 
-class ComicImporterThread(ApplyDBOpsMixin):
+class ComicImporterThread(
+    AggregateMetadataMixin,
+    DeletedMixin,
+    UpdateComicsMixin,
+    FailedImportsMixin,
+    MovedMixin,
+):
     """A worker to handle all bulk database updates."""
 
     def _wait_for_filesystem_ops_to_finish(self, task: UpdaterDBDiffTask) -> bool:
@@ -177,6 +187,13 @@ class ComicImporterThread(ApplyDBOpsMixin):
         self._log_task(library.path, task)
         self._init_librarian_status(task, library.path)
 
+    def _create_comic_relations(self, library, fks):
+        """Query all foreign keys to determine what needs creating, then create them."""
+        if not fks:
+            return 0
+        create_data = self.query_all_missing_fks(library.path, fks)
+        return self.create_all_fks(library, create_data)
+
     def _finish_apply_status(self, library):
         """Finish all librarian statuses."""
         library.update_in_progress = False
@@ -233,24 +250,32 @@ class ComicImporterThread(ApplyDBOpsMixin):
                 "fks": fks,
                 "fis": fis,
             }
-            self.read_metadata(
-                library.path, modified_paths | created_paths, all_metadata
+            self.get_aggregate_metadata(
+                modified_paths | created_paths, library.path, all_metadata
             )
+            all_metadata = None
             modified_paths -= fis.keys()
             created_paths -= fis.keys()
 
-            changed += self.create_comic_relations(library, fks)
+            changed += self._create_comic_relations(library, fks)
+            fks = None
 
-            simple_metadata = {"mds": mds, "m2m_mds": m2m_mds}
-            imported_count = self.update_create_and_link_comics(
-                library, modified_paths, created_paths, simple_metadata
+            imported_count = self.bulk_update_comics(
+                modified_paths,
+                library,
+                created_paths,
+                mds,
             )
+            modified_paths = None
+            imported_count += self.bulk_create_comics(created_paths, library, mds)
+            created_paths = mds = None
+            self.bulk_query_and_link_comic_m2m_fields(m2m_mds)
+            m2m_mds = None
             changed += imported_count
-            all_metadata = (
-                simple_metadata
-            ) = modified_paths = created_paths = mds = m2m_mds = fks = None
 
-            new_failed_imports = self.fail_imports(library, fis)
+            new_failed_imports = self.fail_imports(
+                library, fis, bool(task.files_deleted)
+            )
 
             changed += self.delete(library, task)
             cache.clear()

codex/librarian/importer/link_comics.py

@@ -11,6 +11,7 @@ from codex.models import (
     Imprint,
     Publisher,
     Series,
+    StoryArcNumber,
     Volume,
 )
 from codex.threads import QueuedThread
@@ -50,31 +51,31 @@ class LinkComicsMixin(QueuedThread):
         md["parent_folder"] = Folder.objects.get(path=parent_path)
 
     @staticmethod
-    def _link_folders(folder_paths):
+    def _get_link_folders_filter(folder_paths):
         """Get the ids of all folders to link."""
-        if not folder_paths:
-            return set()
-        folder_pks = Folder.objects.filter(path__in=folder_paths).values_list(
-            "pk", flat=True
-        )
-        return frozenset(folder_pks)
+        return Q(path__in=folder_paths)
 
     @staticmethod
-    def _link_creators(creators_md):
+    def _get_link_creators_filter(creators_md):
         """Get the ids of all creators to link."""
-        if not creators_md:
-            return set()
         creators_filter = Q()
         for creator in creators_md:
-            filter_dict = {
-                "role__name": creator.get("role"),
-                "person__name": creator["person"],
-            }
-            creators_filter |= Q(**filter_dict)
-        creator_pks = Creator.objects.filter(creators_filter).values_list(
-            "pk", flat=True
-        )
-        return frozenset(creator_pks)
+            creators_filter |= Q(
+                role__name=creator.get("role"),
+                person__name=creator["person"],
+            )
+        return creators_filter
+
+    @staticmethod
+    def _get_link_story_arc_numbers_filter(story_arc_numbers_md):
+        """Get the ids of all story_arc_numbers to link."""
+        story_arc_numbers_filter = Q()
+        for name, number in story_arc_numbers_md.items():
+            story_arc_numbers_filter |= Q(
+                story_arc__name=name,
+                number=number,
+            )
+        return story_arc_numbers_filter
 
     def _link_named_m2ms(self, all_m2m_links, comic_pk, md):
         """Set the ids of all named m2m fields into the comic dict."""
@@ -90,6 +91,20 @@ class LinkComicsMixin(QueuedThread):
                 all_m2m_links[field] = {}
             all_m2m_links[field][comic_pk] = frozenset(pks)
 
+    def _link_prepare_special_m2ms(self, link_data, key, model, get_link_filter_method):
+        """Prepare special m2m for linking."""
+        (all_m2m_links, md, comic_pk) = link_data
+        values = md.pop(key, [])
+        if not values:
+            return
+        if key not in all_m2m_links:
+            all_m2m_links[key] = {}
+
+        m2m_filter = get_link_filter_method(values)
+        pks = model.objects.filter(m2m_filter).values_list("pk", flat=True)
+        result = frozenset(pks)
+        all_m2m_links[key][comic_pk] = result
+
     def _link_comic_m2m_fields(self, m2m_mds):
         """Get the complete m2m field data to create."""
         all_m2m_links = {}
@@ -97,17 +112,19 @@ class LinkComicsMixin(QueuedThread):
         comics = Comic.objects.filter(path__in=comic_paths).values_list("pk", "path")
         for comic_pk, comic_path in comics:
             md = m2m_mds[comic_path]
-            if "folders" not in all_m2m_links:
-                all_m2m_links["folders"] = {}
-            try:
-                folder_paths = md.pop("folders")
-            except KeyError:
-                folder_paths = []
-            all_m2m_links["folders"][comic_pk] = self._link_folders(folder_paths)
-            if "creators" not in all_m2m_links:
-                all_m2m_links["creators"] = {}
-            creators_md = md.pop("creators", None)
-            all_m2m_links["creators"][comic_pk] = self._link_creators(creators_md)
+            link_data = (all_m2m_links, md, comic_pk)
+            self._link_prepare_special_m2ms(
+                link_data, "folders", Folder, self._get_link_folders_filter
+            )
+            self._link_prepare_special_m2ms(
+                link_data, "creators", Creator, self._get_link_creators_filter
+            )
+            self._link_prepare_special_m2ms(
+                link_data,
+                "story_arc_numbers",
+                StoryArcNumber,
+                self._get_link_story_arc_numbers_filter,
+            )
             self._link_named_m2ms(all_m2m_links, comic_pk, md)
         return all_m2m_links
 
@@ -125,6 +142,7 @@ class LinkComicsMixin(QueuedThread):
             )
             all_del_pks |= set(del_pks)
             if status:
+                status.total = status.total or 0
                 status.total += len(del_pks)
                 self.status_controller.update(status)
 
@@ -135,6 +153,7 @@ class LinkComicsMixin(QueuedThread):
             )
             missing_pks = set(pks) - extant_pks
             if status:
+                status.total = status.total or 0
                 status.total += len(missing_pks)
                 self.status_controller.update(status)
             for pk in missing_pks:
@@ -165,6 +184,7 @@ class LinkComicsMixin(QueuedThread):
             m2m_links, ThroughModel, through_field_id_name, status
         )
         if status:
+            status.total = status.total or 0
             status.total += len(tms) + len(all_del_pks)
             self.status_controller.update(status)
 
@@ -181,15 +201,18 @@ class LinkComicsMixin(QueuedThread):
                 " relations for altered comics."
             )
         if status:
+            status.complete = status.complete or 0
             status.complete += created_count
             self.status_controller.update(status)
 
         if del_count := len(all_del_pks):
-            ThroughModel.objects.filter(comic_id__in=all_del_pks).delete()
+            del_qs = ThroughModel.objects.filter(pk__in=all_del_pks)
+            del_qs.delete()
             self.log.info(
                 f"Deleted {del_count} stale {field_name} relations for altered comics.",
             )
         if status:
+            status.complete = status.complete or 0
             status.complete += del_count
             self.status_controller.update(status)