codex 1.5.19__py3-none-any.whl → 1.6.0__py3-none-any.whl

codex/librarian/importer/status.py

@@ -1,13 +1,7 @@
 """Librarian Status for imports."""
 
-from collections.abc import Callable
-from enum import Enum
-from time import time
-
 from django.db.models import Choices
 
-from codex.status import Status
-
 
 class ImportStatusTypes(Choices):
     """Keys for Import tasks."""
@@ -25,36 +19,11 @@ class ImportStatusTypes(Choices):
     DIRS_DELETED = "IDD"
     FILES_DELETED = "IFD"
     FAILED_IMPORTS = "IFI"
-
-
-def status_notify(status_type: str | Enum = "", updates=True):
-    """Wrap a function with status changes."""
-    # https://stackoverflow.com/questions/5929107/decorators-with-parameters
-
-    def decorator(func) -> Callable[..., int]:
-        def wrapper(self, data, *args, status=None, **kwargs) -> int:
-            """Run a function bracketed by status changes."""
-            num_elements = len(data)
-            if not num_elements:
-                return 0
-
-            if status:
-                finish = False
-            else:
-                complete = 0 if updates else None
-                status = Status(status_type, complete, num_elements, time())
-                self.status_controller.start(status)
-                finish = True
-
-            kwargs["status"] = status
-            try:
-                count = func(self, data, *args, **kwargs)
-            finally:
-                if finish:
-                    self.status_controller.finish(status)
-
-            return count
-
-        return wrapper
-
-    return decorator
+    QUERY_MISSING_COVERS = "ICQ"
+    COVERS_MOVED = "ICM"
+    COVERS_MODIFIED = "ICU"
+    COVERS_CREATED = "ICC"
+    COVERS_DELETED = "ICD"
+    COVERS_LINK = "ICL"
+    GROUP_UPDATE = "IGU"
+    ADOPT_FOLDERS = "IAF"

codex/librarian/importer/tasks.py

@@ -1,6 +1,8 @@
 """DB Import Tasks."""
 
-from dataclasses import dataclass
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from datetime import datetime
 
 
 @dataclass
@@ -13,14 +15,22 @@ class ImportDBDiffTask(ImportTask):
     """For sending to the updater."""
 
     library_id: int
-    dirs_moved: dict[str, str]
-    files_moved: dict[str, str]
-    dirs_modified: frozenset[str]
-    files_modified: frozenset[str]
-    # dirs_created
-    files_created: frozenset[str]
-    dirs_deleted: frozenset[str]
-    files_deleted: frozenset[str]
+
+    dirs_moved: Mapping[str, str] = field(default_factory=dict)
+    dirs_modified: frozenset[str] = frozenset()
+    # dirs_created: frozenset[str] | None = frozenset()
+    dirs_deleted: frozenset[str] = frozenset()
+
+    files_moved: Mapping[str, str] = field(default_factory=dict)
+    files_modified: frozenset[str] = frozenset()
+    files_created: frozenset[str] = frozenset()
+    files_deleted: frozenset[str] = frozenset()
+
+    covers_moved: Mapping[str, str] = field(default_factory=dict)
+    covers_modified: frozenset[str] = frozenset()
+    covers_created: frozenset[str] = frozenset()
+    covers_deleted: frozenset[str] = frozenset()
+
     force_import_metadata: bool = False
 
 
@@ -34,3 +44,10 @@ class LazyImportComicsTask(ImportTask):
 @dataclass
 class AdoptOrphanFoldersTask(ImportTask):
     """Move orphaned folders into a correct tree position."""
+
+
+@dataclass
+class UpdateGroupsTask(ImportTask):
+    """Force the update of group timestamp."""
+
+    start_time: datetime | None = None

codex/librarian/janitor/cleanup.py

@@ -1,61 +1,84 @@
 """Clean up the database after moves or imports."""
 
 import logging
+from pathlib import Path
 from time import time
+from types import MappingProxyType
 
 from django.contrib.sessions.models import Session
-from django.utils.timezone import now
+from django.db.models.functions.datetime import Now
 
 from codex.librarian.janitor.status import JanitorStatusTypes
 from codex.models import (
+    AgeRating,
     Character,
     Contributor,
     ContributorPerson,
     ContributorRole,
+    Country,
     Folder,
     Genre,
+    Identifier,
+    IdentifierType,
     Imprint,
+    Language,
     Location,
+    OriginalFormat,
     Publisher,
+    ScanInfo,
     Series,
     SeriesGroup,
     StoryArc,
     StoryArcNumber,
     Tag,
+    Tagger,
     Team,
     Volume,
 )
+from codex.models.paths import CustomCover
 from codex.status import Status
 from codex.worker_base import WorkerBaseMixin
 
 _COMIC_FK_CLASSES = (
-    Volume,
-    Series,
-    Imprint,
-    Publisher,
-    Folder,
+    AgeRating,
+    Country,
     Contributor,
-    Tag,
-    Team,
     Character,
+    Genre,
+    Folder,
+    Language,
     Location,
+    Identifier,
+    Imprint,
+    OriginalFormat,
+    Publisher,
+    Series,
     SeriesGroup,
+    ScanInfo,
     StoryArcNumber,
-    Genre,
+    Tagger,
+    Tag,
+    Team,
+    Volume,
 )
 _CONTRIBUTOR_FK_CLASSES = (ContributorRole, ContributorPerson)
 _STORY_ARC_NUMBER_FK_CLASSES = (StoryArc,)
+_IDENTIFIER_FK_CLASSES = (IdentifierType,)
 TOTAL_NUM_FK_CLASSES = (
     len(_COMIC_FK_CLASSES)
     + len(_CONTRIBUTOR_FK_CLASSES)
     + len(_STORY_ARC_NUMBER_FK_CLASSES)
+    + len(_IDENTIFIER_FK_CLASSES)
 )
 
-CLEANUP_MAP = [
-    (_COMIC_FK_CLASSES, "comic"),
-    (_CONTRIBUTOR_FK_CLASSES, "contributor"),
-    (_STORY_ARC_NUMBER_FK_CLASSES, "storyarcnumber"),
-]
+CLEANUP_MAP = MappingProxyType(
+    {
+        "comic": _COMIC_FK_CLASSES,
+        "contributor": _CONTRIBUTOR_FK_CLASSES,
+        "storyarcnumber": _STORY_ARC_NUMBER_FK_CLASSES,
+        "identifier": _IDENTIFIER_FK_CLASSES,
+    }
+)
 
 
 class CleanupMixin(WorkerBaseMixin):
@@ -79,20 +102,42 @@ class CleanupMixin(WorkerBaseMixin):
         try:
             self.status_controller.start(status)
             self.log.debug("Cleaning up unused foreign keys...")
-            for classes, field_name in CLEANUP_MAP:
+            for field_name, classes in CLEANUP_MAP.items():
                 self._bulk_cleanup_fks(classes, field_name, status)
             level = logging.INFO if status.complete else logging.DEBUG
             self.log.log(level, f"Cleaned up {status.complete} unused foreign keys.")
         finally:
             self.status_controller.finish(status)
 
+    def cleanup_custom_covers(self):
+        """Clean up unused custom covers."""
+        start = time()
+        covers = CustomCover.objects.only("path")
+        status = Status(JanitorStatusTypes.CLEANUP_COVERS, 0, covers.count())
+        delete_pks = []
+        try:
+            self.status_controller.start(status)
+            self.log.debug("Cleaning up db custom covers with no source images...")
+            for cover in covers.iterator():
+                if not Path(cover.path).exists():
+                    delete_pks.append(cover.pk)
+                status.increment_complete()
+            delete_qs = CustomCover.objects.filter(pk__in=delete_pks)
+            count = delete_qs.count()
+            delete_qs.delete()
+            level = logging.INFO if status.complete else logging.DEBUG
+            self.log.log(level, f"Deleted {count} CustomCovers without source images.")
+        finally:
+            until = start + 2
+            self.status_controller.finish(status, until=until)
+
     def cleanup_sessions(self):
         """Delete corrupt sessions."""
         start = time()
         status = Status(JanitorStatusTypes.CLEANUP_SESSIONS)
         try:
             self.status_controller.start(status)
-            count, _ = Session.objects.filter(expire_date__lt=now()).delete()
+            count, _ = Session.objects.filter(expire_date__lt=Now()).delete()
             if count:
                 self.log.info(f"Deleted {count} expired sessions.")
 

codex/librarian/janitor/failed_imports.py

@@ -22,6 +22,6 @@ class UpdateFailedImportsMixin(WorkerBaseMixin):
 
     def force_update_all_failed_imports(self):
         """Force update events for failed imports in every library."""
-        pks = Library.objects.values_list("pk", flat=True)
+        pks = Library.objects.filter(covers_only=False).values_list("pk", flat=True)
         for pk in pks:
             self._force_update_failed_imports(pk)

codex/librarian/janitor/janitor.py

@@ -2,12 +2,17 @@
 
 from codex.librarian.covers.status import CoverStatusTypes
 from codex.librarian.covers.tasks import CoverRemoveOrphansTask
+from codex.librarian.importer.status import ImportStatusTypes
+from codex.librarian.importer.tasks import (
+    AdoptOrphanFoldersTask,
+)
 from codex.librarian.janitor.cleanup import TOTAL_NUM_FK_CLASSES, CleanupMixin
 from codex.librarian.janitor.failed_imports import UpdateFailedImportsMixin
 from codex.librarian.janitor.status import JanitorStatusTypes
 from codex.librarian.janitor.tasks import (
     ForceUpdateAllFailedImportsTask,
     JanitorBackupTask,
+    JanitorCleanCoversTask,
     JanitorCleanFKsTask,
     JanitorCleanupSessionsTask,
     JanitorClearStatusTask,
@@ -30,12 +35,15 @@ from codex.status import Status
 
 _JANITOR_STATII = (
     Status(JanitorStatusTypes.CLEANUP_FK, 0, TOTAL_NUM_FK_CLASSES),
+    Status(JanitorStatusTypes.CLEANUP_COVERS),
     Status(JanitorStatusTypes.CLEANUP_SESSIONS),
     Status(JanitorStatusTypes.DB_OPTIMIZE),
     Status(JanitorStatusTypes.DB_BACKUP),
     Status(JanitorStatusTypes.CODEX_UPDATE),
     Status(CoverStatusTypes.FIND_ORPHAN),
     Status(CoverStatusTypes.PURGE_COVERS),
+    Status(ImportStatusTypes.ADOPT_FOLDERS),
+    Status(ImportStatusTypes.DIRS_MOVED),
     Status(SearchIndexStatusTypes.SEARCH_INDEX_UPDATE),
     Status(SearchIndexStatusTypes.SEARCH_INDEX_REMOVE),
     Status(SearchIndexStatusTypes.SEARCH_INDEX_MERGE),
@@ -59,10 +67,12 @@ class Janitor(CleanupMixin, UpdateMixin, VacuumMixin, UpdateFailedImportsMixin):
             tasks = (
                 SearchIndexAbortTask(),
                 JanitorCleanFKsTask(),
+                JanitorCleanCoversTask(),
                 JanitorCleanupSessionsTask(),
                 JanitorVacuumTask(),
                 JanitorBackupTask(),
                 JanitorUpdateTask(force=False),
+                AdoptOrphanFoldersTask(),
                 CoverRemoveOrphansTask(),
                 SearchIndexUpdateTask(False),
                 SearchIndexMergeTask(optimize),
@@ -73,7 +83,7 @@ class Janitor(CleanupMixin, UpdateMixin, VacuumMixin, UpdateFailedImportsMixin):
         except Exception:
             self.log.exception(f"In {self.__class__.__name__}")
 
-    def run(self, task):
+    def run(self, task):  # noqa: PLR0912,C901
         """Run Janitor tasks as the librarian process directly."""
         try:
             match task:
@@ -89,6 +99,8 @@ class Janitor(CleanupMixin, UpdateMixin, VacuumMixin, UpdateFailedImportsMixin):
                     self.shutdown_codex()
                 case JanitorCleanFKsTask():
                     self.cleanup_fks()
+                case JanitorCleanCoversTask():
+                    self.cleanup_custom_covers()
                 case JanitorCleanupSessionsTask():
                     self.cleanup_sessions()
                 case JanitorClearStatusTask():

codex/librarian/janitor/janitord.py

@@ -1,4 +1,4 @@
-"""Perform maintence tasks."""
+"""Perform maintenance tasks."""
 
 from datetime import datetime, time, timedelta
 from threading import Condition, Event

codex/librarian/janitor/status.py

@@ -13,3 +13,4 @@ class JanitorStatusTypes(Choices):
     DB_OPTIMIZE = "JDO"
     DB_BACKUP = "JDB"
     CLEANUP_SESSIONS = "JSD"
+    CLEANUP_COVERS = "JCD"

codex/librarian/janitor/tasks.py

@@ -40,6 +40,11 @@ class JanitorCleanFKsTask(JanitorTask):
     """Clean unused foreign keys."""
 
 
+@dataclass
+class JanitorCleanCoversTask(JanitorTask):
+    """Clean unused custom covers."""
+
+
 @dataclass
 class JanitorClearStatusTask(JanitorTask):
     """Clear all librarian statuses."""

codex/librarian/librariand.py

@@ -7,11 +7,14 @@ from typing import NamedTuple
 
 from caseconverter import snakecase
 
-from codex.librarian.covers.coverd import CoverContributorThread
+from codex.librarian.covers.coverd import CoverThread
 from codex.librarian.covers.tasks import CoverTask
 from codex.librarian.delayed_taskd import DelayedTasksThread
 from codex.librarian.importer.importerd import ComicImporterThread
-from codex.librarian.importer.tasks import AdoptOrphanFoldersTask, ImportTask
+from codex.librarian.importer.tasks import (
+    AdoptOrphanFoldersTask,
+    ImportTask,
+)
 from codex.librarian.janitor.janitor import Janitor
 from codex.librarian.janitor.janitord import JanitorThread
 from codex.librarian.janitor.tasks import JanitorTask
@@ -43,7 +46,7 @@ class LibrarianDaemon(Process, LoggerBaseMixin):
     _THREAD_CLASSES = (
         NotifierThread,
         DelayedTasksThread,
-        CoverContributorThread,
+        CoverThread,
         SearchIndexerThread,
         ComicImporterThread,
         WatchdogEventBatcherThread,
@@ -73,15 +76,16 @@ class LibrarianDaemon(Process, LoggerBaseMixin):
             WatchdogSyncTask(),
             SearchIndexRebuildIfDBChangedTask(),
         )
+
         for task in startup_tasks:
             self.queue.put(task)
         self.search_indexer_abort_event = Manager().Event()
 
-    def _process_task(self, task):
+    def _process_task(self, task):  # noqa: PLR0912,C901
         """Process an individual task popped off the queue."""
         match task:
             case CoverTask():
-                self._threads.cover_contributor_thread.queue.put(task)
+                self._threads.cover_thread.queue.put(task)
             case WatchdogEventTask():
                 self._threads.watchdog_event_batcher_thread.queue.put(task)
             case ImportTask():

codex/librarian/notifier/notifierd.py

@@ -22,7 +22,7 @@ class NotifierThread(AggregateMessageQueuedThread):
         and do a group_send with it's message.
         """
         item = {
-            "group": task.group.name,
+            "group": task.group,
             "message": {
                 "type": "send_text",
                 "text": task.text,

codex/librarian/notifier/tasks.py

@@ -10,9 +10,9 @@ class NotifierTask:
     """Handle with the Notifier."""
 
     text: str
-    group: ChannelGroups
+    group: str
 
 
-LIBRARY_CHANGED_TASK = NotifierTask("LIBRARY_CHANGED", ChannelGroups.ALL)
-LIBRARIAN_STATUS_TASK = NotifierTask("LIBRARIAN_STATUS", ChannelGroups.ADMIN)
-FAILED_IMPORTS_TASK = NotifierTask("FAILED_IMPORTS", ChannelGroups.ADMIN)
+LIBRARY_CHANGED_TASK = NotifierTask("LIBRARY_CHANGED", ChannelGroups.ALL.name)
+LIBRARIAN_STATUS_TASK = NotifierTask("LIBRARIAN_STATUS", ChannelGroups.ADMIN.name)
+FAILED_IMPORTS_TASK = NotifierTask("FAILED_IMPORTS", ChannelGroups.ADMIN.name)

codex/librarian/search/update.py

@@ -319,7 +319,7 @@ class UpdateMixin(RemoveMixin):
         self.abort_event.clear()
         try:
             any_update_in_progress = Library.objects.filter(
-                update_in_progress=True
+                covers_only=False, update_in_progress=True
             ).exists()
             if any_update_in_progress:
                 self.log.debug(

codex/librarian/watchdog/db_snapshot.py

@@ -7,19 +7,20 @@ from pathlib import Path
 from watchdog.utils.dirsnapshot import DirectorySnapshot
 
 from codex.logger_base import LoggerBaseMixin
-from codex.models import Comic, FailedImport, Folder
+from codex.models import Comic, CustomCover, FailedImport, Folder
 
 
 class CodexDatabaseSnapshot(DirectorySnapshot, LoggerBaseMixin):
     """Take snapshots from the Codex database."""
 
-    MODELS = (Folder, Comic, FailedImport)
+    MODELS = (Folder, Comic, FailedImport, CustomCover)
+    COVERS_ONLY_MODELS = (CustomCover,)
     _STAT_LEN = 10
 
-    @classmethod
-    def _walk(cls, root):
+    def _walk(self, root):
         """Populate the DirectorySnapshot structures from the database."""
-        for model in cls.MODELS:
+        models = self.COVERS_ONLY_MODELS if self._covers_only else self.MODELS
+        for model in models:
             yield (
                 model.objects.filter(library__path=root)
                 .order_by("path")
@@ -57,7 +58,7 @@ class CodexDatabaseSnapshot(DirectorySnapshot, LoggerBaseMixin):
         i = (st.st_ino, st.st_dev)
         self._inode_to_path[i] = path
 
-    def __init__(
+    def __init__(  # noqa: PLR0913
         self,
         path,
         _recursive=True,  # unused, always recursive
@@ -65,8 +66,10 @@ class CodexDatabaseSnapshot(DirectorySnapshot, LoggerBaseMixin):
         _listdir=os.listdir,  # unused for database
         force=False,
         log_queue=None,
+        covers_only=False,
     ):
         """Initialize like DirectorySnapshot but use a database walk."""
+        self._covers_only = covers_only
         self.init_logger(log_queue)
         self._stat_info = {}
         self._inode_to_path = {}

codex/librarian/watchdog/emitter.py

@@ -54,6 +54,7 @@ class DatabasePollingEmitter(EventEmitter, WorkerBaseMixin):
         timeout=DEFAULT_EMITTER_TIMEOUT,
         log_queue=None,
         librarian_queue=None,
+        covers_only=False,
     ):
         """Initialize snapshot methods."""
         self.init_worker(log_queue, librarian_queue)
@@ -61,6 +62,7 @@ class DatabasePollingEmitter(EventEmitter, WorkerBaseMixin):
         self._force = False
         self._watch_path = Path(watch.path)
         self._watch_path_unmounted = self._watch_path / _DOCKER_UNMOUNTED_FN
+        self._covers_only = covers_only
         super().__init__(
             event_queue, watch, timeout=timeout, event_filter=_CODEX_EVENT_FILTER
         )
@@ -84,6 +86,7 @@ class DatabasePollingEmitter(EventEmitter, WorkerBaseMixin):
             self.watch.is_recursive,
             force=self._force,
             log_queue=self.log_queue,
+            covers_only=self._covers_only,
         )
         self._force = False
         return db_snapshot

codex/librarian/watchdog/event_batcherd.py

@@ -25,12 +25,16 @@ class WatchdogEventBatcherThread(AggregateMessageQueuedThread):
         {
             "library_id": None,
             "dirs_moved": {},
+            "dirs_modified": set(),
+            "dirs_deleted": set(),
             "files_moved": {},
             "files_modified": set(),
             "files_created": set(),
-            "dirs_deleted": set(),
-            "dirs_modified": set(),
             "files_deleted": set(),
+            "covers_moved": {},
+            "covers_modified": set(),
+            "covers_created": set(),
+            "covers_deleted": set(),
         }
     )
     MAX_DELAY = 60
@@ -54,7 +58,13 @@ class WatchdogEventBatcherThread(AggregateMessageQueuedThread):
         """Translate event class names into field names."""
         self._ensure_library_args(library_id)
 
-        prefix = "dir" if event.is_directory else "file"
+        prefix = (
+            "dir"
+            if event.is_directory
+            else "cover"
+            if getattr(event, "is_cover", False)
+            else "file"
+        )
         field = f"{prefix}s_{event.event_type}"
 
         return self.cache[library_id].get(field)
@@ -90,17 +100,23 @@ class WatchdogEventBatcherThread(AggregateMessageQueuedThread):
         for src_path in args["files_deleted"]:
             with suppress(KeyError):
                 del args["files_moved"][src_path]
+
         # created
         args["files_created"] -= args["files_deleted"]
         files_dest_paths = set(args["files_moved"].values())
         args["files_created"] -= files_dest_paths
+
         # modified
         args["dirs_modified"] -= args["dirs_deleted"]
         args["dirs_modified"] -= set(args["dirs_moved"].values())
+
         args["files_modified"] -= args["files_created"]
         args["files_modified"] -= args["files_deleted"]
         args["files_modified"] -= files_dest_paths
 
+        args["covers_modified"] -= args["covers_deleted"]
+        args["covers_modified"] -= args["covers_created"]
+
     def _create_task(self, library_id):
         """Create a task from cached aggregated message data."""
         self._deduplicate_events(library_id)