codex 1.6.0__py3-none-any.whl → 1.6.0a1__py3-none-any.whl

codex/librarian/importer/update_comics.py

@@ -0,0 +1,89 @@
+"""Bulk update comic objects."""
+
+from django.db.models import NOT_PROVIDED
+from django.db.models.functions import Now
+
+from codex.librarian.covers.tasks import CoverRemoveTask
+from codex.librarian.importer.const import (
+    BULK_UPDATE_COMIC_FIELDS,
+    BULK_UPDATE_COMIC_FIELDS_WITH_VALUES,
+)
+from codex.librarian.importer.link_comics import LinkComicsMixin
+from codex.librarian.importer.status import ImportStatusTypes, status_notify
+from codex.models import (
+    Comic,
+    Timestamp,
+)
+
+
+class UpdateComicsMixin(LinkComicsMixin):
+    """Create comics methods."""
+
+    def _bulk_update_comics_add_comic(self, static_data, comic, results):
+        """Add one comic and stats to the bulk update list."""
+        try:
+            library, mds, now = static_data
+            md = mds.pop(comic.path)
+            self.get_comic_fk_links(md, library, comic.path)
+            for field_name in BULK_UPDATE_COMIC_FIELDS_WITH_VALUES:
+                value = md.get(field_name)
+                if value is None:
+                    default_value = Comic._meta.get_field(field_name).default
+                    if default_value != NOT_PROVIDED:
+                        value = default_value
+                setattr(comic, field_name, value)
+            comic.presave()
+            comic.updated_at = now
+            update_comics, comic_pks, comic_update_paths = results
+            update_comics.append(comic)
+            comic_pks.append(comic.pk)
+            comic_update_paths.add(comic.path)
+        except Exception:
+            self.log.exception(f"Error preparing {comic} for update.")
+
+    @status_notify(status_type=ImportStatusTypes.FILES_MODIFIED, updates=False)
+    def bulk_update_comics(self, comic_paths, library, create_paths, mds, **kwargs):
+        """Bulk update comics, and move nonextant comics into create job.."""
+        count = 0
+        if not comic_paths:
+            return count
+
+        self.log.debug(
+            f"Preparing {len(comic_paths)} comics for update in library {library.path}."
+        )
+        # Get existing comics to update
+        comics = Comic.objects.filter(library=library, path__in=comic_paths).only(
+            *BULK_UPDATE_COMIC_FIELDS
+        )
+
+        # set attributes for each comic
+        update_comics = []
+        now = Now()
+        comic_pks = []
+        comic_update_paths = set()
+        static_data = library, mds, now
+        results = update_comics, comic_pks, comic_update_paths
+        for comic in comics.iterator():
+            self._bulk_update_comics_add_comic(static_data, comic, results)
+
+        converted_create_paths = frozenset(set(comic_paths) - comic_update_paths)
+        create_paths.update(converted_create_paths)
+        count = len(converted_create_paths)
+        if count:
+            self.log.info(f"Converted {count} update paths to create paths.")
+
+        self.log.debug(f"Bulk updating {len(update_comics)} comics.")
+        try:
+            Comic.objects.bulk_update(update_comics, BULK_UPDATE_COMIC_FIELDS)
+            Timestamp.touch(Timestamp.TimestampChoices.COVERS)
+            count = len(update_comics)
+
+            task = CoverRemoveTask(frozenset(comic_pks))
+            self.log.debug(f"Purging covers for {len(comic_pks)} updated comics.")
+            self.librarian_queue.put(task)
+            if count:
+                self.log.info(f"Updated {count} comics.")
+        except Exception:
+            self.log.exception(f"While updating {comic_update_paths}")
+
+        return count

codex/librarian/janitor/cleanup.py

@@ -1,12 +1,11 @@
 """Clean up the database after moves or imports."""
 
 import logging
-from pathlib import Path
 from time import time
 from types import MappingProxyType
 
 from django.contrib.sessions.models import Session
-from django.db.models.functions.datetime import Now
+from django.utils.timezone import now
 
 from codex.librarian.janitor.status import JanitorStatusTypes
 from codex.models import (
@@ -35,7 +34,6 @@ from codex.models import (
     Team,
     Volume,
 )
-from codex.models.paths import CustomCover
 from codex.status import Status
 from codex.worker_base import WorkerBaseMixin
 
@@ -109,35 +107,13 @@ class CleanupMixin(WorkerBaseMixin):
         finally:
             self.status_controller.finish(status)
 
-    def cleanup_custom_covers(self):
-        """Clean up unused custom covers."""
-        start = time()
-        covers = CustomCover.objects.only("path")
-        status = Status(JanitorStatusTypes.CLEANUP_COVERS, 0, covers.count())
-        delete_pks = []
-        try:
-            self.status_controller.start(status)
-            self.log.debug("Cleaning up db custom covers with no source images...")
-            for cover in covers.iterator():
-                if not Path(cover.path).exists():
-                    delete_pks.append(cover.pk)
-                status.increment_complete()
-            delete_qs = CustomCover.objects.filter(pk__in=delete_pks)
-            count = delete_qs.count()
-            delete_qs.delete()
-            level = logging.INFO if status.complete else logging.DEBUG
-            self.log.log(level, f"Deleted {count} CustomCovers without source images.")
-        finally:
-            until = start + 2
-            self.status_controller.finish(status, until=until)
-
     def cleanup_sessions(self):
         """Delete corrupt sessions."""
         start = time()
         status = Status(JanitorStatusTypes.CLEANUP_SESSIONS)
         try:
             self.status_controller.start(status)
-            count, _ = Session.objects.filter(expire_date__lt=Now()).delete()
+            count, _ = Session.objects.filter(expire_date__lt=now()).delete()
             if count:
                 self.log.info(f"Deleted {count} expired sessions.")
 

codex/librarian/janitor/failed_imports.py

@@ -22,6 +22,6 @@ class UpdateFailedImportsMixin(WorkerBaseMixin):
 
     def force_update_all_failed_imports(self):
         """Force update events for failed imports in every library."""
-        pks = Library.objects.filter(covers_only=False).values_list("pk", flat=True)
+        pks = Library.objects.values_list("pk", flat=True)
         for pk in pks:
             self._force_update_failed_imports(pk)

codex/librarian/janitor/janitor.py

@@ -2,17 +2,12 @@
 
 from codex.librarian.covers.status import CoverStatusTypes
 from codex.librarian.covers.tasks import CoverRemoveOrphansTask
-from codex.librarian.importer.status import ImportStatusTypes
-from codex.librarian.importer.tasks import (
-    AdoptOrphanFoldersTask,
-)
 from codex.librarian.janitor.cleanup import TOTAL_NUM_FK_CLASSES, CleanupMixin
 from codex.librarian.janitor.failed_imports import UpdateFailedImportsMixin
 from codex.librarian.janitor.status import JanitorStatusTypes
 from codex.librarian.janitor.tasks import (
     ForceUpdateAllFailedImportsTask,
     JanitorBackupTask,
-    JanitorCleanCoversTask,
     JanitorCleanFKsTask,
     JanitorCleanupSessionsTask,
     JanitorClearStatusTask,
@@ -35,15 +30,12 @@ from codex.status import Status
 
 _JANITOR_STATII = (
     Status(JanitorStatusTypes.CLEANUP_FK, 0, TOTAL_NUM_FK_CLASSES),
-    Status(JanitorStatusTypes.CLEANUP_COVERS),
     Status(JanitorStatusTypes.CLEANUP_SESSIONS),
     Status(JanitorStatusTypes.DB_OPTIMIZE),
     Status(JanitorStatusTypes.DB_BACKUP),
     Status(JanitorStatusTypes.CODEX_UPDATE),
     Status(CoverStatusTypes.FIND_ORPHAN),
     Status(CoverStatusTypes.PURGE_COVERS),
-    Status(ImportStatusTypes.ADOPT_FOLDERS),
-    Status(ImportStatusTypes.DIRS_MOVED),
     Status(SearchIndexStatusTypes.SEARCH_INDEX_UPDATE),
     Status(SearchIndexStatusTypes.SEARCH_INDEX_REMOVE),
     Status(SearchIndexStatusTypes.SEARCH_INDEX_MERGE),
@@ -67,12 +59,10 @@ class Janitor(CleanupMixin, UpdateMixin, VacuumMixin, UpdateFailedImportsMixin):
             tasks = (
                 SearchIndexAbortTask(),
                 JanitorCleanFKsTask(),
-                JanitorCleanCoversTask(),
                 JanitorCleanupSessionsTask(),
                 JanitorVacuumTask(),
                 JanitorBackupTask(),
                 JanitorUpdateTask(force=False),
-                AdoptOrphanFoldersTask(),
                 CoverRemoveOrphansTask(),
                 SearchIndexUpdateTask(False),
                 SearchIndexMergeTask(optimize),
@@ -83,7 +73,7 @@ class Janitor(CleanupMixin, UpdateMixin, VacuumMixin, UpdateFailedImportsMixin):
         except Exception:
             self.log.exception(f"In {self.__class__.__name__}")
 
-    def run(self, task):  # noqa: PLR0912,C901
+    def run(self, task):
         """Run Janitor tasks as the librarian process directly."""
         try:
             match task:
@@ -99,8 +89,6 @@ class Janitor(CleanupMixin, UpdateMixin, VacuumMixin, UpdateFailedImportsMixin):
                     self.shutdown_codex()
                 case JanitorCleanFKsTask():
                     self.cleanup_fks()
-                case JanitorCleanCoversTask():
-                    self.cleanup_custom_covers()
                 case JanitorCleanupSessionsTask():
                     self.cleanup_sessions()
                 case JanitorClearStatusTask():

codex/librarian/janitor/janitord.py

@@ -1,4 +1,4 @@
-"""Perform maintenance tasks."""
+"""Perform maintence tasks."""
 
 from datetime import datetime, time, timedelta
 from threading import Condition, Event

codex/librarian/janitor/status.py

@@ -13,4 +13,3 @@ class JanitorStatusTypes(Choices):
     DB_OPTIMIZE = "JDO"
     DB_BACKUP = "JDB"
     CLEANUP_SESSIONS = "JSD"
-    CLEANUP_COVERS = "JCD"

codex/librarian/janitor/tasks.py

@@ -40,11 +40,6 @@ class JanitorCleanFKsTask(JanitorTask):
     """Clean unused foreign keys."""
 
 
-@dataclass
-class JanitorCleanCoversTask(JanitorTask):
-    """Clean unused custom covers."""
-
-
 @dataclass
 class JanitorClearStatusTask(JanitorTask):
     """Clear all librarian statuses."""

codex/librarian/librariand.py

@@ -7,14 +7,11 @@ from typing import NamedTuple
 
 from caseconverter import snakecase
 
-from codex.librarian.covers.coverd import CoverThread
+from codex.librarian.covers.coverd import CoverContributorThread
 from codex.librarian.covers.tasks import CoverTask
 from codex.librarian.delayed_taskd import DelayedTasksThread
 from codex.librarian.importer.importerd import ComicImporterThread
-from codex.librarian.importer.tasks import (
-    AdoptOrphanFoldersTask,
-    ImportTask,
-)
+from codex.librarian.importer.tasks import AdoptOrphanFoldersTask, ImportTask
 from codex.librarian.janitor.janitor import Janitor
 from codex.librarian.janitor.janitord import JanitorThread
 from codex.librarian.janitor.tasks import JanitorTask
@@ -46,7 +43,7 @@ class LibrarianDaemon(Process, LoggerBaseMixin):
     _THREAD_CLASSES = (
         NotifierThread,
         DelayedTasksThread,
-        CoverThread,
+        CoverContributorThread,
         SearchIndexerThread,
         ComicImporterThread,
         WatchdogEventBatcherThread,
@@ -76,16 +73,15 @@ class LibrarianDaemon(Process, LoggerBaseMixin):
             WatchdogSyncTask(),
             SearchIndexRebuildIfDBChangedTask(),
         )
-
         for task in startup_tasks:
             self.queue.put(task)
         self.search_indexer_abort_event = Manager().Event()
 
-    def _process_task(self, task):  # noqa: PLR0912,C901
+    def _process_task(self, task):
         """Process an individual task popped off the queue."""
         match task:
             case CoverTask():
-                self._threads.cover_thread.queue.put(task)
+                self._threads.cover_contributor_thread.queue.put(task)
             case WatchdogEventTask():
                 self._threads.watchdog_event_batcher_thread.queue.put(task)
             case ImportTask():

codex/librarian/notifier/notifierd.py

@@ -22,7 +22,7 @@ class NotifierThread(AggregateMessageQueuedThread):
         and do a group_send with it's message.
         """
         item = {
-            "group": task.group,
+            "group": task.group.name,
             "message": {
                 "type": "send_text",
                 "text": task.text,

codex/librarian/notifier/tasks.py

@@ -10,9 +10,9 @@ class NotifierTask:
     """Handle with the Notifier."""
 
     text: str
-    group: str
+    group: ChannelGroups
 
 
-LIBRARY_CHANGED_TASK = NotifierTask("LIBRARY_CHANGED", ChannelGroups.ALL.name)
-LIBRARIAN_STATUS_TASK = NotifierTask("LIBRARIAN_STATUS", ChannelGroups.ADMIN.name)
-FAILED_IMPORTS_TASK = NotifierTask("FAILED_IMPORTS", ChannelGroups.ADMIN.name)
+LIBRARY_CHANGED_TASK = NotifierTask("LIBRARY_CHANGED", ChannelGroups.ALL)
+LIBRARIAN_STATUS_TASK = NotifierTask("LIBRARIAN_STATUS", ChannelGroups.ADMIN)
+FAILED_IMPORTS_TASK = NotifierTask("FAILED_IMPORTS", ChannelGroups.ADMIN)

codex/librarian/search/update.py

@@ -319,7 +319,7 @@ class UpdateMixin(RemoveMixin):
         self.abort_event.clear()
         try:
             any_update_in_progress = Library.objects.filter(
-                covers_only=False, update_in_progress=True
+                update_in_progress=True
             ).exists()
             if any_update_in_progress:
                 self.log.debug(

codex/librarian/watchdog/db_snapshot.py

@@ -7,20 +7,19 @@ from pathlib import Path
 from watchdog.utils.dirsnapshot import DirectorySnapshot
 
 from codex.logger_base import LoggerBaseMixin
-from codex.models import Comic, CustomCover, FailedImport, Folder
+from codex.models import Comic, FailedImport, Folder
 
 
 class CodexDatabaseSnapshot(DirectorySnapshot, LoggerBaseMixin):
     """Take snapshots from the Codex database."""
 
-    MODELS = (Folder, Comic, FailedImport, CustomCover)
-    COVERS_ONLY_MODELS = (CustomCover,)
+    MODELS = (Folder, Comic, FailedImport)
     _STAT_LEN = 10
 
-    def _walk(self, root):
+    @classmethod
+    def _walk(cls, root):
         """Populate the DirectorySnapshot structures from the database."""
-        models = self.COVERS_ONLY_MODELS if self._covers_only else self.MODELS
-        for model in models:
+        for model in cls.MODELS:
             yield (
                 model.objects.filter(library__path=root)
                 .order_by("path")
@@ -58,7 +57,7 @@ class CodexDatabaseSnapshot(DirectorySnapshot, LoggerBaseMixin):
         i = (st.st_ino, st.st_dev)
         self._inode_to_path[i] = path
 
-    def __init__(  # noqa: PLR0913
+    def __init__(
         self,
         path,
         _recursive=True,  # unused, always recursive
@@ -66,10 +65,8 @@ class CodexDatabaseSnapshot(DirectorySnapshot, LoggerBaseMixin):
         _listdir=os.listdir,  # unused for database
         force=False,
         log_queue=None,
-        covers_only=False,
     ):
         """Initialize like DirectorySnapshot but use a database walk."""
-        self._covers_only = covers_only
         self.init_logger(log_queue)
         self._stat_info = {}
         self._inode_to_path = {}

codex/librarian/watchdog/emitter.py

@@ -54,7 +54,6 @@ class DatabasePollingEmitter(EventEmitter, WorkerBaseMixin):
         timeout=DEFAULT_EMITTER_TIMEOUT,
         log_queue=None,
         librarian_queue=None,
-        covers_only=False,
     ):
         """Initialize snapshot methods."""
         self.init_worker(log_queue, librarian_queue)
@@ -62,7 +61,6 @@ class DatabasePollingEmitter(EventEmitter, WorkerBaseMixin):
         self._force = False
         self._watch_path = Path(watch.path)
         self._watch_path_unmounted = self._watch_path / _DOCKER_UNMOUNTED_FN
-        self._covers_only = covers_only
         super().__init__(
             event_queue, watch, timeout=timeout, event_filter=_CODEX_EVENT_FILTER
         )
@@ -86,7 +84,6 @@ class DatabasePollingEmitter(EventEmitter, WorkerBaseMixin):
             self.watch.is_recursive,
             force=self._force,
             log_queue=self.log_queue,
-            covers_only=self._covers_only,
         )
         self._force = False
         return db_snapshot

codex/librarian/watchdog/event_batcherd.py

@@ -25,16 +25,12 @@ class WatchdogEventBatcherThread(AggregateMessageQueuedThread):
         {
             "library_id": None,
             "dirs_moved": {},
-            "dirs_modified": set(),
-            "dirs_deleted": set(),
             "files_moved": {},
             "files_modified": set(),
             "files_created": set(),
+            "dirs_deleted": set(),
+            "dirs_modified": set(),
             "files_deleted": set(),
-            "covers_moved": {},
-            "covers_modified": set(),
-            "covers_created": set(),
-            "covers_deleted": set(),
         }
     )
     MAX_DELAY = 60
@@ -58,13 +54,7 @@ class WatchdogEventBatcherThread(AggregateMessageQueuedThread):
         """Translate event class names into field names."""
         self._ensure_library_args(library_id)
 
-        prefix = (
-            "dir"
-            if event.is_directory
-            else "cover"
-            if getattr(event, "is_cover", False)
-            else "file"
-        )
+        prefix = "dir" if event.is_directory else "file"
         field = f"{prefix}s_{event.event_type}"
 
         return self.cache[library_id].get(field)
@@ -100,23 +90,17 @@ class WatchdogEventBatcherThread(AggregateMessageQueuedThread):
         for src_path in args["files_deleted"]:
             with suppress(KeyError):
                 del args["files_moved"][src_path]
-
         # created
         args["files_created"] -= args["files_deleted"]
         files_dest_paths = set(args["files_moved"].values())
         args["files_created"] -= files_dest_paths
-
         # modified
         args["dirs_modified"] -= args["dirs_deleted"]
         args["dirs_modified"] -= set(args["dirs_moved"].values())
-
         args["files_modified"] -= args["files_created"]
         args["files_modified"] -= args["files_deleted"]
         args["files_modified"] -= files_dest_paths
 
-        args["covers_modified"] -= args["covers_deleted"]
-        args["covers_modified"] -= args["covers_created"]
-
     def _create_task(self, library_id):
         """Create a task from cached aggregated message data."""
         self._deduplicate_events(library_id)