codex 1.6.0a6__py3-none-any.whl → 1.6.0a8__py3-none-any.whl

codex/librarian/importer/init.py

@@ -0,0 +1,265 @@
+"""Initiale Importer."""
+
+import logging
+from pathlib import Path
+from time import sleep, time
+from typing import Any
+
+from django.utils.timezone import now
+
+from codex.librarian.importer.status import ImportStatusTypes
+from codex.librarian.importer.tasks import ImportDBDiffTask
+from codex.librarian.search.status import SearchIndexStatusTypes
+from codex.librarian.search.tasks import SearchIndexAbortTask
+from codex.models import Library
+from codex.status import Status
+from codex.worker_base import WorkerBaseMixin
+
+_WRITE_WAIT_EXPIRY = 60
+
+
+class InitImporter(WorkerBaseMixin):
+    """Initiale Importer."""
+
+    def __init__(self, task: ImportDBDiffTask, log_queue, librarian_queue):
+        """Initialize the import."""
+        self.init_worker(log_queue, librarian_queue)
+        self.task: ImportDBDiffTask = task
+        self.metadata: dict[str, Any] = {}
+        self.changed: int = 0
+        self.library = Library.objects.only("path", "update_in_progress").get(
+            pk=self.task.library_id
+        )
+
+    def _wait_for_filesystem_ops_to_finish(self) -> bool:
+        """Watchdog sends events before filesystem events finish, so wait for them."""
+        started_checking = time()
+
+        # Don't wait for deletes to complete.
+        # Do wait for move, modified, create files before import.
+        all_modified_paths = (
+            frozenset(self.task.dirs_moved.values())
+            | frozenset(self.task.files_moved.values())
+            | self.task.dirs_modified
+            | self.task.files_modified
+            | self.task.files_created
+        )
+
+        old_total_size = -1
+        total_size = 0
+        wait_time = 2
+        while old_total_size != total_size:
+            if old_total_size > 0:
+                # second time around or more
+                sleep(wait_time)
+                wait_time = wait_time**2
+                self.log.debug(
+                    f"Waiting for files to copy before import: "
+                    f"{old_total_size} != {total_size}"
+                )
+            if time() - started_checking > _WRITE_WAIT_EXPIRY:
+                return True
+
+            old_total_size = total_size
+            total_size = 0
+            for path_str in all_modified_paths:
+                path = Path(path_str)
+                if path.exists():
+                    total_size += Path(path).stat().st_size
+        return False
+
+    #######
+    # LOG #
+    #######
+    def _log_task_construct_dirs_log(self):
+        """Construct dirs log line."""
+        dirs_log = []
+        if self.task.dirs_moved:
+            dirs_log += [f"{len(self.task.dirs_moved)} moved"]
+        if self.task.dirs_modified:
+            dirs_log += [f"{len(self.task.dirs_modified)} modified"]
+        if self.task.dirs_deleted:
+            dirs_log += [f"{len(self.task.dirs_deleted)} deleted"]
+        return dirs_log
+
+    def _log_task_construct_comics_log(self):
+        """Construct comcis log line."""
+        comics_log = []
+        if self.task.files_moved:
+            comics_log += [f"{len(self.task.files_moved)} moved"]
+        if self.task.files_modified:
+            comics_log += [f"{len(self.task.files_modified)} modified"]
+        if self.task.files_created:
+            comics_log += [f"{len(self.task.files_created)} created"]
+        if self.task.files_deleted:
+            comics_log += [f"{len(self.task.files_deleted)} deleted"]
+        return comics_log
+
+    def _log_task(self):
+        """Log the watchdog event self.task."""
+        if self.log.getEffectiveLevel() < logging.DEBUG:
+            return
+
+        self.log.debug(f"Updating library {self.library.path}...")
+        comics_log = self._log_task_construct_comics_log()
+        if comics_log:
+            log = "Comics: "
+            log += ", ".join(comics_log)
+            self.log.debug("  " + log)
+
+        dirs_log = self._log_task_construct_dirs_log()
+        if dirs_log:
+            log = "Folders: "
+            log += ", ".join(dirs_log)
+            self.log.debug("  " + log)
+
+    ########
+    # INIT #
+    ########
+    def _init_librarian_status_moved(self, status_list):
+        """Initialize moved statuses."""
+        search_index_updates = 0
+        if self.task.dirs_moved:
+            status_list += [
+                Status(ImportStatusTypes.DIRS_MOVED, None, len(self.task.dirs_moved))
+            ]
+        if self.task.files_moved:
+            status_list += [
+                Status(ImportStatusTypes.FILES_MOVED, None, len(self.task.files_moved))
+            ]
+            search_index_updates += len(self.task.files_moved)
+        if self.task.covers_moved:
+            status_list += [
+                Status(
+                    ImportStatusTypes.COVERS_MOVED, None, len(self.task.covers_moved)
+                )
+            ]
+        if self.task.dirs_modified:
+            status_list += [
+                Status(
+                    ImportStatusTypes.DIRS_MODIFIED, None, len(self.task.dirs_modified)
+                )
+            ]
+        return search_index_updates
+
+    def _init_if_modified_or_created(self, path, status_list):
+        """Initialize librarian statuses for modified or created ops."""
+        total_paths = len(self.task.files_modified) + len(self.task.files_created)
+        status_list += [
+            Status(ImportStatusTypes.AGGREGATE_TAGS, 0, total_paths, subtitle=path),
+            Status(ImportStatusTypes.QUERY_MISSING_FKS),
+            Status(ImportStatusTypes.QUERY_MISSING_COVERS),
+            Status(ImportStatusTypes.CREATE_FKS),
+        ]
+        if self.task.files_modified:
+            status_list += [
+                Status(
+                    ImportStatusTypes.FILES_MODIFIED,
+                    None,
+                    len(self.task.files_modified),
+                )
+            ]
+        if self.task.covers_modified:
+            status_list += [
+                Status(
+                    ImportStatusTypes.COVERS_MODIFIED,
+                    None,
+                    len(self.task.covers_modified),
+                )
+            ]
+
+        if self.task.files_created or self.task.covers_created:
+            status_list += [
+                Status(
+                    ImportStatusTypes.FILES_CREATED, None, len(self.task.files_created)
+                )
+            ]
+        if self.task.covers_created:
+            status_list += [
+                Status(
+                    ImportStatusTypes.COVERS_CREATED,
+                    None,
+                    len(self.task.covers_created),
+                )
+            ]
+
+        if self.task.files_modified or self.task.files_created:
+            status_list += [Status(ImportStatusTypes.LINK_M2M_FIELDS)]
+
+        num_covers_linked = (
+            len(self.task.covers_moved)
+            + len(self.task.covers_modified)
+            + len(self.task.covers_created)
+        )
+        if num_covers_linked:
+            status_list += [
+                Status(ImportStatusTypes.COVERS_LINK, None, num_covers_linked)
+            ]
+
+        return total_paths
+
+    def _init_librarian_status_deleted(self, status_list):
+        """Init deleted statuses."""
+        search_index_updates = 0
+        if self.task.files_deleted:
+            status_list += [
+                Status(
+                    ImportStatusTypes.FILES_DELETED, None, len(self.task.files_deleted)
+                )
+            ]
+            search_index_updates += len(self.task.files_deleted)
+        if self.task.covers_deleted:
+            status_list += [
+                Status(
+                    ImportStatusTypes.COVERS_DELETED,
+                    None,
+                    len(self.task.covers_deleted),
+                )
+            ]
+        return search_index_updates
+
+    @staticmethod
+    def _init_librarian_status_search_index(search_index_updates, status_list):
+        """Init search index statuses."""
+        status_list += [
+            Status(
+                SearchIndexStatusTypes.SEARCH_INDEX_UPDATE,
+                0,
+                search_index_updates,
+            ),
+            Status(SearchIndexStatusTypes.SEARCH_INDEX_REMOVE),
+        ]
+
+    def _init_librarian_status(self, path):
+        """Update the librarian status self.tasks."""
+        status_list = []
+        search_index_updates = 0
+        search_index_updates += self._init_librarian_status_moved(status_list)
+        if (
+            self.task.files_modified
+            or self.task.files_created
+            or self.task.covers_modified
+            or self.task.covers_created
+        ):
+            search_index_updates += self._init_if_modified_or_created(path, status_list)
+        search_index_updates += self._init_librarian_status_deleted(status_list)
+        status_list += [Status(ImportStatusTypes.GROUP_UPDATE)]
+        self._init_librarian_status_search_index(search_index_updates, status_list)
+        self.status_controller.start_many(status_list)
+
+    def init_apply(self):
+        """Initialize the library and status flags."""
+        self.start_time = now()
+        self.librarian_queue.put(SearchIndexAbortTask())
+        self.library.update_in_progress = True
+        self.library.save()
+        too_long = self._wait_for_filesystem_ops_to_finish()
+        if too_long:
+            self.log.warning(
+                "Import apply waited for the filesystem to stop changing too long. "
+                "Try polling again once files have finished copying"
+                f" in library: {self.library.path}"
+            )
+            return
+        self._log_task()
+        self._init_librarian_status(self.library.path)

codex/librarian/importer/link_comics.py

@@ -5,12 +5,12 @@ from pathlib import Path
 from django.db.models import Q
 
 from codex.librarian.importer.const import (
-    CLASS_CUSTOM_COVER_GROUP_MAP,
     COMIC_FK_FIELD_NAMES,
     DICT_MODEL_FIELD_NAME_CLASS_MAP,
     DICT_MODEL_REL_LINK_MAP,
     FOLDERS_FIELD,
     IMPRINT,
+    M2M_MDS,
     PARENT_FOLDER,
     PUBLISHER,
     SERIES,
@@ -18,10 +18,10 @@ from codex.librarian.importer.const import (
     STORY_ARCS_METADATA_KEY,
     VOLUME,
 )
-from codex.librarian.importer.status import ImportStatusTypes, status_notify
+from codex.librarian.importer.link_covers import LinkCoversImporter
+from codex.librarian.importer.status import ImportStatusTypes
 from codex.models import (
     Comic,
-    CustomCover,
     Folder,
     Imprint,
     Publisher,
@@ -29,10 +29,10 @@ from codex.models import (
     Volume,
 )
 from codex.models.named import Identifier
-from codex.threads import QueuedThread
+from codex.status import Status
 
 
-class LinkComicsMixin(QueuedThread):
+class LinkComicsImporter(LinkCoversImporter):
     """Link comics methods."""
 
     @staticmethod
@@ -42,13 +42,12 @@ class LinkComicsMixin(QueuedThread):
         return md.get(field_name, group_class.DEFAULT_NAME)
 
     @classmethod
-    def get_comic_fk_links(cls, md, library, path):
+    def get_comic_fk_links(cls, md, path):
         """Get links for all foreign keys for creating and updating."""
         publisher_name = cls._get_group_name(Publisher, md)
         imprint_name = cls._get_group_name(Imprint, md)
         series_name = cls._get_group_name(Series, md)
         volume_name = cls._get_group_name(Volume, md)
-        md["library"] = library
         md[PUBLISHER] = Publisher.objects.get(name=publisher_name)
         md[IMPRINT] = Imprint.objects.get(
             publisher__name=publisher_name, name=imprint_name
@@ -124,13 +123,15 @@ class LinkComicsMixin(QueuedThread):
         result = frozenset(pks)
         all_m2m_links[key][comic_pk] = result
 
-    def _link_comic_m2m_fields(self, m2m_mds):
+    def _link_comic_m2m_fields(self):
         """Get the complete m2m field data to create."""
         all_m2m_links = {}
-        comic_paths = frozenset(m2m_mds.keys())
+        comic_paths = frozenset(self.metadata.get(M2M_MDS, {}).keys())
+        if not comic_paths:
+            return all_m2m_links
         comics = Comic.objects.filter(path__in=comic_paths).values_list("pk", "path")
         for comic_pk, comic_path in comics:
-            md = m2m_mds[comic_path]
+            md = self.metadata[M2M_MDS][comic_path]
             link_data = (all_m2m_links, md, comic_pk)
             self._link_prepare_special_m2ms(
                 link_data, FOLDERS_FIELD, Folder, self._get_link_folders_filter
@@ -143,6 +144,7 @@ class LinkComicsMixin(QueuedThread):
                 self._link_prepare_special_m2ms(link_data, field_name, model, method)
 
             self._link_named_m2ms(all_m2m_links, comic_pk, md)
+        self.metadata.pop(M2M_MDS)
         return all_m2m_links
 
     def _query_relation_adjustments(
@@ -204,10 +206,9 @@ class LinkComicsMixin(QueuedThread):
         tms, all_del_pks = self._query_relation_adjustments(
             m2m_links, ThroughModel, through_field_id_name, status
         )
-        if status:
-            status.total = status.total or 0
-            status.total += len(tms) + len(all_del_pks)
-            self.status_controller.update(status)
+        status.total = status.total or 0
+        status.total += len(tms) + len(all_del_pks)
+        self.status_controller.update(status)
 
         update_fields = ("comic_id", through_field_id_name)
         ThroughModel.objects.bulk_create(
@@ -218,10 +219,9 @@ class LinkComicsMixin(QueuedThread):
         )
         if created_count := len(tms):
             self.log.info(f"Linked {created_count} new {field_name} to altered comics.")
-        if status:
-            status.complete = status.complete or 0
-            status.complete += created_count
-            self.status_controller.update(status)
+        status.complete = status.complete or 0
+        status.complete += created_count
+        self.status_controller.update(status)
 
         if del_count := len(all_del_pks):
             del_qs = ThroughModel.objects.filter(pk__in=all_del_pks)
@@ -229,19 +229,16 @@ class LinkComicsMixin(QueuedThread):
             self.log.info(
                 f"Deleted {del_count} stale {field_name} relations for altered comics.",
             )
-        if status:
-            status.complete = status.complete or 0
-            status.complete += del_count
-            self.status_controller.update(status)
+        status.complete = status.complete or 0
+        status.complete += del_count
+        self.status_controller.update(status)
         return created_count, del_count
 
-    @status_notify(status_type=ImportStatusTypes.LINK_M2M_FIELDS, updates=False)
-    def bulk_query_and_link_comic_m2m_fields(self, all_m2m_mds, status=None):
+    def bulk_query_and_link_comic_m2m_fields(self):
         """Combine query and bulk link into a batch."""
-        if status:
-            status.complete = 0
-            status.total = 0
-        all_m2m_links = self._link_comic_m2m_fields(all_m2m_mds)
+        status = Status(ImportStatusTypes.LINK_M2M_FIELDS)
+        self.status_controller.start(status)
+        all_m2m_links = self._link_comic_m2m_fields()
         created_total = 0
         del_total = 0
         for field_name, m2m_links in all_m2m_links.items():
@@ -258,57 +255,5 @@ class LinkComicsMixin(QueuedThread):
             self.log.info(f"Linked {created_total} comics to tags.")
         if del_total:
             self.log.info(f"Deleted {del_total} stale relations for altered comics.")
+        self.status_controller.finish(status)
         return created_total + del_total
-
-    def _link_custom_cover_prepare(self, cover, model_map):
-        """Prepare one cover in the model map for bulk update."""
-        if cover.library and cover.library.covers_only:
-            model = CLASS_CUSTOM_COVER_GROUP_MAP.inverse.get(cover.group)
-            if not model:
-                self.log.warning(f"Custom Cover model not found for {cover.path}")
-                return
-            group_filter = {"sort_name": cover.sort_name}
-        else:
-            model = Folder
-            path = str(Path(cover.path).parent)
-            group_filter = {"path": path}
-        qs = model.objects.filter(**group_filter).exclude(custom_cover=cover)
-        if not qs.exists():
-            return
-
-        if model not in model_map:
-            model_map[model] = []
-
-        for obj in qs.iterator():
-            obj.custom_cover = cover
-            model_map[model].append(obj)
-
-    def _link_custom_cover_group(self, model, objs, status):
-        """Bulk link a group to it's custom covers."""
-        count = 0
-        if not objs:
-            return count
-        model.objects.bulk_update(objs, ["custom_cover"])
-        count += len(objs)
-        self.log.info(f"Linked {count} custom covers to {model.__name__}s")
-        if status:
-            status.complete = status.complete or 0
-            status.complete += count
-        return count
-
-    @status_notify(status_type=ImportStatusTypes.COVERS_LINK)
-    def link_custom_covers(self, link_cover_pks, status=None):
-        """Link Custom Covers to Groups."""
-        # Aggregate objs to update for each group model.
-        model_map = {}
-        covers = CustomCover.objects.filter(pk__in=link_cover_pks).only(
-            "library", "path"
-        )
-        for cover in covers:
-            self._link_custom_cover_prepare(cover, model_map)
-
-        # Bulk update each model type
-        total_count = 0
-        for model, objs in model_map.items():
-            total_count += self._link_custom_cover_group(model, objs, status)
-        return total_count

codex/librarian/importer/link_covers.py

@@ -0,0 +1,73 @@
+"""Link Covers."""
+
+from pathlib import Path
+
+from codex.librarian.importer.const import CLASS_CUSTOM_COVER_GROUP_MAP, LINK_COVER_PKS
+from codex.librarian.importer.failed_imports import FailedImportsImporter
+from codex.librarian.importer.status import ImportStatusTypes
+from codex.models import CustomCover, Folder
+from codex.status import Status
+
+
+class LinkCoversImporter(FailedImportsImporter):
+    """Link Covers methods."""
+
+    def _link_custom_cover_prepare(self, cover, model_map):
+        """Prepare one cover in the model map for bulk update."""
+        if cover.library and cover.library.covers_only:
+            model = CLASS_CUSTOM_COVER_GROUP_MAP.inverse.get(cover.group)
+            if not model:
+                self.log.warning(f"Custom Cover model not found for {cover.path}")
+                return
+            group_filter = {"sort_name": cover.sort_name}
+        else:
+            model = Folder
+            path = str(Path(cover.path).parent)
+            group_filter = {"path": path}
+        qs = model.objects.filter(**group_filter).exclude(custom_cover=cover)
+        if not qs.exists():
+            return
+
+        if model not in model_map:
+            model_map[model] = []
+
+        for obj in qs.iterator():
+            obj.custom_cover = cover
+            model_map[model].append(obj)
+
+    def _link_custom_cover_group(self, model, objs, status):
+        """Bulk link a group to it's custom covers."""
+        count = 0
+        if not objs:
+            return count
+        model.objects.bulk_update(objs, ["custom_cover"])
+        count += len(objs)
+        self.log.info(f"Linked {count} custom covers to {model.__name__}s")
+        status.complete += count
+        self.status_controller.update(status, notify=False)
+        return count
+
+    def link_custom_covers(self):
+        """Link Custom Covers to Groups."""
+        link_cover_pks = self.metadata.get(LINK_COVER_PKS, {})
+        num_link_cover_pks = len(link_cover_pks)
+        if not num_link_cover_pks:
+            return 0
+        status = Status(ImportStatusTypes.COVERS_LINK, 0, num_link_cover_pks)
+        try:
+            self.status_controller.start(status)
+            # Aggregate objs to update for each group model.
+            model_map = {}
+            covers = CustomCover.objects.filter(pk__in=link_cover_pks).only(
+                "library", "path"
+            )
+            for cover in covers:
+                self._link_custom_cover_prepare(cover, model_map)
+
+            # Bulk update each model type
+            total_count = 0
+            for model, objs in model_map.items():
+                total_count += self._link_custom_cover_group(model, objs, status)
+        finally:
+            self.status_controller.finish(status)
+        return total_count