codex 1.6.0a4__py3-none-any.whl → 1.6.0a5__py3-none-any.whl

codex/librarian/importer/query_fks.py

@@ -1,5 +1,6 @@
 """Query the missing foreign keys for comics and contributors."""
 
+from itertools import chain
 from logging import DEBUG, INFO
 from pathlib import Path
 from types import MappingProxyType
@@ -35,6 +36,7 @@ from codex.models import (
     Volume,
 )
 from codex.models.named import Identifier
+from codex.models.paths import CustomCover
 from codex.settings.settings import FILTER_BATCH_SIZE
 from codex.status import Status
 from codex.threads import QueuedThread
@@ -566,6 +568,30 @@ class QueryForeignKeysMixin(QueuedThread):
             self.log.info(f"Prepared {count} new Folders.")
         return count
 
+    @staticmethod
+    def _get_create_fks_totals(create_data):
+        (
+            create_groups,
+            update_groups,
+            create_folder_paths,
+            create_fks,
+            create_contributors,
+            create_story_arc_numbers,
+            create_identifiers,
+        ) = create_data
+        total_fks = 0
+        for data_group in chain(
+            create_groups.values(), update_groups.values(), create_fks.values()
+        ):
+            total_fks += len(data_group)
+        total_fks += (
+            len(create_folder_paths)
+            + len(create_contributors)
+            + len(create_story_arc_numbers)
+            + len(create_identifiers)
+        )
+        return total_fks
+
     def query_all_missing_fks(self, library_path, fks):
         """Get objects to create by querying existing objects for the proposed fks."""
         create_contributors = set()
@@ -619,7 +645,7 @@ class QueryForeignKeysMixin(QueuedThread):
         finally:
             self.status_controller.finish(status)
 
-        return (
+        create_data = (
             create_groups,
             update_groups,
             create_folder_paths,
@@ -628,3 +654,41 @@ class QueryForeignKeysMixin(QueuedThread):
             create_story_arc_numbers,
             create_identifiers,
         )
+
+        total_fks = self._get_create_fks_totals(create_data)
+        status = Status(ImportStatusTypes.CREATE_FKS, 0, total_fks)
+        self.status_controller.update(status, notify=False)
+        return (*create_data, total_fks)
+
+    @status_notify(ImportStatusTypes.QUERY_MISSING_COVERS, updates=False)
+    def query_missing_custom_covers(
+        self, cover_paths: frozenset, library, query_data: list, status=None
+    ):
+        """Identify update & create covers."""
+        self.log.debug(f"Querying {len(cover_paths)} custom cover_paths")
+        if status:
+            status.total = len(cover_paths)
+
+        update_covers_qs = CustomCover.objects.filter(
+            library=library, path__in=cover_paths
+        )
+        query_data.append(update_covers_qs)
+        update_cover_paths = frozenset(update_covers_qs.values_list("path", flat=True))
+        update_count = len(update_cover_paths)
+        update_status = Status(ImportStatusTypes.COVERS_MODIFIED, 0, update_count)
+        self.status_controller.update(update_status, notify=False)
+
+        create_cover_paths = cover_paths - update_cover_paths
+        query_data.append(create_cover_paths)
+        create_count = len(create_cover_paths)
+        create_status = Status(ImportStatusTypes.COVERS_CREATED, 0, create_count)
+        self.status_controller.update(create_status, notify=False)
+
+        count = create_count + update_count
+        if count:
+            self.log.debug(
+                f"Discovered {update_count} custom covers to update and {create_count} to create."
+            )
+            if status:
+                status.add_complete(count)
+        return count

codex/librarian/importer/status.py

@@ -25,6 +25,12 @@ class ImportStatusTypes(Choices):
     DIRS_DELETED = "IDD"
     FILES_DELETED = "IFD"
     FAILED_IMPORTS = "IFI"
+    QUERY_MISSING_COVERS = "ICQ"
+    COVERS_MOVED = "ICM"
+    COVERS_MODIFIED = "ICU"
+    COVERS_CREATED = "ICC"
+    COVERS_DELETED = "ICD"
+    COVERS_LINK = "ICL"
 
 
 def status_notify(status_type: str | Enum = "", updates=True):

codex/librarian/importer/tasks.py

@@ -13,14 +13,22 @@ class ImportDBDiffTask(ImportTask):
     """For sending to the updater."""
 
     library_id: int
+
     dirs_moved: dict[str, str]
-    files_moved: dict[str, str]
     dirs_modified: frozenset[str]
+    # dirs_created: frozenset[str]
+    dirs_deleted: frozenset[str]
+
+    files_moved: dict[str, str]
     files_modified: frozenset[str]
-    # dirs_created
     files_created: frozenset[str]
-    dirs_deleted: frozenset[str]
     files_deleted: frozenset[str]
+
+    covers_moved: dict[str, str]
+    covers_modified: frozenset[str]
+    covers_created: frozenset[str]
+    covers_deleted: frozenset[str]
+
     force_import_metadata: bool = False
 
 

codex/librarian/importer/update_comics.py

@@ -3,7 +3,6 @@
 from django.db.models import NOT_PROVIDED
 from django.db.models.functions import Now
 
-from codex.librarian.covers.tasks import CoverRemoveTask
 from codex.librarian.importer.const import (
     BULK_UPDATE_COMIC_FIELDS,
     BULK_UPDATE_COMIC_FIELDS_WITH_VALUES,
@@ -78,9 +77,8 @@ class UpdateComicsMixin(LinkComicsMixin):
             Timestamp.touch(Timestamp.TimestampChoices.COVERS)
             count = len(update_comics)
 
-            task = CoverRemoveTask(frozenset(comic_pks))
+            self._remove_covers(comic_pks, False)  # type: ignore
             self.log.debug(f"Purging covers for {len(comic_pks)} updated comics.")
-            self.librarian_queue.put(task)
             if count:
                 self.log.info(f"Updated {count} comics.")
         except Exception:

codex/librarian/janitor/cleanup.py

@@ -1,6 +1,7 @@
 """Clean up the database after moves or imports."""
 
 import logging
+from pathlib import Path
 from time import time
 from types import MappingProxyType
 
@@ -34,6 +35,7 @@ from codex.models import (
     Team,
     Volume,
 )
+from codex.models.paths import CustomCover
 from codex.status import Status
 from codex.worker_base import WorkerBaseMixin
 
@@ -107,6 +109,28 @@ class CleanupMixin(WorkerBaseMixin):
         finally:
             self.status_controller.finish(status)
 
+    def cleanup_custom_covers(self):
+        """Clean up unused custom covers."""
+        start = time()
+        covers = CustomCover.objects.only("path")
+        status = Status(JanitorStatusTypes.CLEANUP_COVERS, 0, covers.count())
+        delete_pks = []
+        try:
+            self.status_controller.start(status)
+            self.log.debug("Cleaning up db custom covers with no source images...")
+            for cover in covers.iterator():
+                if not Path(cover.path).exists():
+                    delete_pks.append(cover.pk)
+                status.increment_complete()
+            delete_qs = CustomCover.objects.filter(pk__in=delete_pks)
+            count = delete_qs.count()
+            delete_qs.delete()
+            level = logging.INFO if status.complete else logging.DEBUG
+            self.log.log(level, f"Deleted {count} CustomCovers without source images.")
+        finally:
+            until = start + 2
+            self.status_controller.finish(status, until=until)
+
     def cleanup_sessions(self):
         """Delete corrupt sessions."""
         start = time()

codex/librarian/janitor/failed_imports.py

@@ -22,6 +22,6 @@ class UpdateFailedImportsMixin(WorkerBaseMixin):
 
     def force_update_all_failed_imports(self):
         """Force update events for failed imports in every library."""
-        pks = Library.objects.values_list("pk", flat=True)
+        pks = Library.objects.filter(covers_only=False).values_list("pk", flat=True)
         for pk in pks:
             self._force_update_failed_imports(pk)

codex/librarian/janitor/janitor.py

@@ -8,6 +8,7 @@ from codex.librarian.janitor.status import JanitorStatusTypes
 from codex.librarian.janitor.tasks import (
     ForceUpdateAllFailedImportsTask,
     JanitorBackupTask,
+    JanitorCleanCoversTask,
     JanitorCleanFKsTask,
     JanitorCleanupSessionsTask,
     JanitorClearStatusTask,
@@ -30,6 +31,7 @@ from codex.status import Status
 
 _JANITOR_STATII = (
     Status(JanitorStatusTypes.CLEANUP_FK, 0, TOTAL_NUM_FK_CLASSES),
+    Status(JanitorStatusTypes.CLEANUP_COVERS),
     Status(JanitorStatusTypes.CLEANUP_SESSIONS),
     Status(JanitorStatusTypes.DB_OPTIMIZE),
     Status(JanitorStatusTypes.DB_BACKUP),
@@ -59,6 +61,7 @@ class Janitor(CleanupMixin, UpdateMixin, VacuumMixin, UpdateFailedImportsMixin):
             tasks = (
                 SearchIndexAbortTask(),
                 JanitorCleanFKsTask(),
+                JanitorCleanCoversTask(),
                 JanitorCleanupSessionsTask(),
                 JanitorVacuumTask(),
                 JanitorBackupTask(),
@@ -89,6 +92,8 @@ class Janitor(CleanupMixin, UpdateMixin, VacuumMixin, UpdateFailedImportsMixin):
                     self.shutdown_codex()
                 case JanitorCleanFKsTask():
                     self.cleanup_fks()
+                case JanitorCleanCoversTask():
+                    self.cleanup_custom_covers()
                 case JanitorCleanupSessionsTask():
                     self.cleanup_sessions()
                 case JanitorClearStatusTask():

codex/librarian/janitor/janitord.py

@@ -1,4 +1,4 @@
-"""Perform maintence tasks."""
+"""Perform maintenance tasks."""
 
 from datetime import datetime, time, timedelta
 from threading import Condition, Event

codex/librarian/janitor/status.py

@@ -13,3 +13,4 @@ class JanitorStatusTypes(Choices):
     DB_OPTIMIZE = "JDO"
     DB_BACKUP = "JDB"
     CLEANUP_SESSIONS = "JSD"
+    CLEANUP_COVERS = "JCD"

codex/librarian/janitor/tasks.py

@@ -40,6 +40,11 @@ class JanitorCleanFKsTask(JanitorTask):
     """Clean unused foreign keys."""
 
 
+@dataclass
+class JanitorCleanCoversTask(JanitorTask):
+    """Clean unused custom covers."""
+
+
 @dataclass
 class JanitorClearStatusTask(JanitorTask):
     """Clear all librarian statuses."""

codex/librarian/librariand.py

@@ -7,7 +7,7 @@ from typing import NamedTuple
 
 from caseconverter import snakecase
 
-from codex.librarian.covers.coverd import CoverContributorThread
+from codex.librarian.covers.coverd import CoverThread
 from codex.librarian.covers.tasks import CoverTask
 from codex.librarian.delayed_taskd import DelayedTasksThread
 from codex.librarian.importer.importerd import ComicImporterThread
@@ -43,7 +43,7 @@ class LibrarianDaemon(Process, LoggerBaseMixin):
     _THREAD_CLASSES = (
         NotifierThread,
         DelayedTasksThread,
-        CoverContributorThread,
+        CoverThread,
         SearchIndexerThread,
         ComicImporterThread,
         WatchdogEventBatcherThread,
@@ -81,7 +81,7 @@ class LibrarianDaemon(Process, LoggerBaseMixin):
         """Process an individual task popped off the queue."""
         match task:
             case CoverTask():
-                self._threads.cover_contributor_thread.queue.put(task)
+                self._threads.cover_thread.queue.put(task)
             case WatchdogEventTask():
                 self._threads.watchdog_event_batcher_thread.queue.put(task)
             case ImportTask():

codex/librarian/search/update.py

@@ -319,7 +319,7 @@ class UpdateMixin(RemoveMixin):
         self.abort_event.clear()
         try:
             any_update_in_progress = Library.objects.filter(
-                update_in_progress=True
+                covers_only=False, update_in_progress=True
             ).exists()
             if any_update_in_progress:
                 self.log.debug(

codex/librarian/watchdog/db_snapshot.py

@@ -7,19 +7,20 @@ from pathlib import Path
 from watchdog.utils.dirsnapshot import DirectorySnapshot
 
 from codex.logger_base import LoggerBaseMixin
-from codex.models import Comic, FailedImport, Folder
+from codex.models import Comic, CustomCover, FailedImport, Folder
 
 
 class CodexDatabaseSnapshot(DirectorySnapshot, LoggerBaseMixin):
     """Take snapshots from the Codex database."""
 
-    MODELS = (Folder, Comic, FailedImport)
+    MODELS = (Folder, Comic, FailedImport, CustomCover)
+    COVERS_ONLY_MODELS = (CustomCover,)
     _STAT_LEN = 10
 
-    @classmethod
-    def _walk(cls, root):
+    def _walk(self, root):
         """Populate the DirectorySnapshot structures from the database."""
-        for model in cls.MODELS:
+        models = self.COVERS_ONLY_MODELS if self._covers_only else self.MODELS
+        for model in models:
             yield (
                 model.objects.filter(library__path=root)
                 .order_by("path")
@@ -57,7 +58,7 @@ class CodexDatabaseSnapshot(DirectorySnapshot, LoggerBaseMixin):
         i = (st.st_ino, st.st_dev)
         self._inode_to_path[i] = path
 
-    def __init__(
+    def __init__(  # noqa: PLR0913
         self,
         path,
         _recursive=True,  # unused, always recursive
@@ -65,8 +66,10 @@ class CodexDatabaseSnapshot(DirectorySnapshot, LoggerBaseMixin):
         _listdir=os.listdir,  # unused for database
         force=False,
         log_queue=None,
+        covers_only=False,
     ):
         """Initialize like DirectorySnapshot but use a database walk."""
+        self._covers_only = covers_only
         self.init_logger(log_queue)
         self._stat_info = {}
         self._inode_to_path = {}

codex/librarian/watchdog/emitter.py

@@ -54,6 +54,7 @@ class DatabasePollingEmitter(EventEmitter, WorkerBaseMixin):
         timeout=DEFAULT_EMITTER_TIMEOUT,
         log_queue=None,
         librarian_queue=None,
+        covers_only=False,
     ):
         """Initialize snapshot methods."""
         self.init_worker(log_queue, librarian_queue)
@@ -61,6 +62,7 @@ class DatabasePollingEmitter(EventEmitter, WorkerBaseMixin):
         self._force = False
         self._watch_path = Path(watch.path)
         self._watch_path_unmounted = self._watch_path / _DOCKER_UNMOUNTED_FN
+        self._covers_only = covers_only
         super().__init__(
             event_queue, watch, timeout=timeout, event_filter=_CODEX_EVENT_FILTER
         )
@@ -84,6 +86,7 @@ class DatabasePollingEmitter(EventEmitter, WorkerBaseMixin):
             self.watch.is_recursive,
             force=self._force,
             log_queue=self.log_queue,
+            covers_only=self._covers_only,
         )
         self._force = False
         return db_snapshot

codex/librarian/watchdog/event_batcherd.py

@@ -25,12 +25,16 @@ class WatchdogEventBatcherThread(AggregateMessageQueuedThread):
         {
             "library_id": None,
             "dirs_moved": {},
+            "dirs_modified": set(),
+            "dirs_deleted": set(),
             "files_moved": {},
             "files_modified": set(),
             "files_created": set(),
-            "dirs_deleted": set(),
-            "dirs_modified": set(),
             "files_deleted": set(),
+            "covers_moved": {},
+            "covers_modified": set(),
+            "covers_created": set(),
+            "covers_deleted": set(),
         }
     )
     MAX_DELAY = 60
@@ -54,7 +58,13 @@ class WatchdogEventBatcherThread(AggregateMessageQueuedThread):
         """Translate event class names into field names."""
         self._ensure_library_args(library_id)
 
-        prefix = "dir" if event.is_directory else "file"
+        prefix = (
+            "dir"
+            if event.is_directory
+            else "cover"
+            if getattr(event, "is_cover", False)
+            else "file"
+        )
         field = f"{prefix}s_{event.event_type}"
 
         return self.cache[library_id].get(field)
@@ -90,17 +100,23 @@ class WatchdogEventBatcherThread(AggregateMessageQueuedThread):
         for src_path in args["files_deleted"]:
             with suppress(KeyError):
                 del args["files_moved"][src_path]
+
         # created
         args["files_created"] -= args["files_deleted"]
         files_dest_paths = set(args["files_moved"].values())
         args["files_created"] -= files_dest_paths
+
         # modified
         args["dirs_modified"] -= args["dirs_deleted"]
         args["dirs_modified"] -= set(args["dirs_moved"].values())
+
         args["files_modified"] -= args["files_created"]
         args["files_modified"] -= args["files_deleted"]
         args["files_modified"] -= files_dest_paths
 
+        args["covers_modified"] -= args["covers_deleted"]
+        args["covers_modified"] -= args["covers_created"]
+
     def _create_task(self, library_id):
         """Create a task from cached aggregated message data."""
         self._deduplicate_events(library_id)