codex 1.7.14__py3-none-any.whl → 1.8.0a2__py3-none-any.whl

codex/librarian/bookmark/bookmarkd.py

@@ -3,13 +3,21 @@
 from collections.abc import Mapping
 from dataclasses import dataclass
 
+from loguru import logger
+from typing_extensions import override
+
+from codex.librarian.bookmark.latest_version import CodexLatestVersionUpdater
 from codex.librarian.bookmark.tasks import (
     BookmarkUpdateTask,
+    ClearLibrarianStatusTask,
+    CodexLatestVersionTask,
     UserActiveTask,
 )
-from codex.librarian.bookmark.update import BookmarkUpdate
+from codex.librarian.bookmark.update import BookmarkUpdateMixin
 from codex.librarian.bookmark.user_active import UserActiveMixin
-from codex.threads import AggregateMessageQueuedThread
+from codex.librarian.telemeter.tasks import TelemeterTask
+from codex.librarian.telemeter.telemeter import send_telemetry
+from codex.librarian.threads import AggregateMessageQueuedThread
 
 
 @dataclass
@@ -20,6 +28,7 @@ class BookmarkKey:
     comic_pks: tuple = ()
     user_pk: int = 0
 
+    @override
     def __hash__(self):
         """Hash the dict as a tuple."""
         auth_filters = (
@@ -27,43 +36,73 @@ class BookmarkKey:
         )
         return hash((auth_filters, self.comic_pks, self.user_pk))
 
+    @override
     def __eq__(self, other):
         """Equal uses hashes."""
         return self.__hash__() == other.__hash__()
 
 
 class BookmarkThread(
-    UserActiveMixin,
     AggregateMessageQueuedThread,
-    BookmarkUpdate,
+    BookmarkUpdateMixin,
+    UserActiveMixin,
 ):
     """Aggregates Bookmark updates preventing floods updates db in batches.."""
 
     FLOOD_DELAY = 3.0
     MAX_DELAY = 5.0
 
+    def __init__(self, *args, **kwargs):
+        """Init mixins."""
+        super().__init__(*args, **kwargs)
+        self.init_group_acl()
+        self.init_user_active()
+
+    def _process_task_immediately(self, task):
+        if self.db_write_lock.locked():
+            self.log.warning(f"Database locked, not processing {task}")
+        match task:
+            case TelemeterTask():
+                send_telemetry(self.log)
+            case ClearLibrarianStatusTask():
+                self.status_controller.finish_many([])
+            case CodexLatestVersionTask():
+                worker = CodexLatestVersionUpdater(
+                    self.log, self.librarian_queue, self.db_write_lock
+                )
+                worker.update_latest_version(force=task.force)
+            case _:
+                self.log.warning(f"Unknown Bookmark task {task}")
+
+    @override
     def aggregate_items(self, item):
         """Aggregate bookmark updates."""
-        if isinstance(item, UserActiveTask):
-            # Wedge the user active recorer into the bookmark thread because it
-            # it also wants to be done offline and low priority.
-            key = BookmarkKey(user_pk=item.pk)
-            self.cache[key] = None
-        elif isinstance(item, BookmarkUpdateTask):
-            key = BookmarkKey(item.auth_filter, item.comic_pks)
-            if key not in self.cache:
-                self.cache[key] = {}
-            self.cache[key].update(item.updates)
-        else:
-            self.log.warning(f"Unknown Bookmark task {item}")
+        task = item
+        match task:
+            case UserActiveTask():
+                # Wedge the user active recorer into the bookmark thread because it
+                # it also wants to be done offline and low priority.
+                key = BookmarkKey(user_pk=item.pk)
+                self.cache[key] = None
+            case BookmarkUpdateTask():
+                key = BookmarkKey(item.auth_filter, item.comic_pks)
+                if key not in self.cache:
+                    self.cache[key] = {}
+                self.cache[key].update(item.updates)
+            case _:
+                self._process_task_immediately(task)
 
+    @override
     def send_all_items(self):
         """Run the task method."""
+        if self.db_write_lock.locked():
+            self.log.debug("Database locked, waiting to process bookmarks.")
+            return
         cleanup = set()
         for key, value in self.cache.items():
             try:
                 if key.user_pk:
-                    self.update_user_active(key.user_pk, self.log)
+                    self.update_user_active(key.user_pk, logger)
                 elif key.comic_pks:
                     self.update_bookmarks(key.auth_filter, key.comic_pks, value)
                 cleanup.add(key)

codex/librarian/bookmark/latest_version.py

@@ -0,0 +1,65 @@
+"""Fetch the current codex version."""
+
+import json
+from datetime import timedelta
+
+import requests
+from django.utils import timezone
+
+from codex.librarian.scribe.janitor.status import JanitorCodexLatestVersionStatus
+from codex.librarian.scribe.janitor.tasks import JanitorCodexUpdateTask
+from codex.librarian.worker import WorkerStatusMixin
+from codex.models import Timestamp
+from codex.version import PACKAGE_NAME
+
+_PYPI_URL_TEMPLATE = "https://pypi.python.org/pypi/%s/json"
+_REPO_URL = _PYPI_URL_TEMPLATE % PACKAGE_NAME
+_CACHE_EXPIRY = timedelta(days=1) - timedelta(minutes=5)
+_REPO_TIMEOUT = 5
+
+
+class CodexLatestVersionUpdater(WorkerStatusMixin):
+    """Methods for fetching the latest version."""
+
+    @staticmethod
+    def _fetch_latest_version():
+        """Fetch Latest Remotely."""
+        response = requests.get(_REPO_URL, timeout=_REPO_TIMEOUT)
+        return json.loads(response.text)["info"]["version"]
+
+    def update_latest_version(self, *, force: bool, update: bool = False):
+        """Get the latest version from a remote repo using a cache."""
+        if self.db_write_lock.locked():
+            self.log.warning("Database locked, not retrieving latest codex version.")
+            return
+        status = JanitorCodexLatestVersionStatus()
+        try:
+            self.status_controller.start(status)
+            ts = Timestamp.objects.get(key=Timestamp.Choices.CODEX_VERSION.value)
+            do_fetch = (
+                force
+                or not ts.version
+                or (timezone.now() - ts.updated_at > _CACHE_EXPIRY)
+            )
+            if do_fetch:
+                latest_version = self._fetch_latest_version()
+                if not latest_version:
+                    reason = "Bad latest version fetched."
+                    raise ValueError(reason)
+                ts.version = latest_version
+                ts.save()
+                level = "INFO"
+                log_txt = f"Saved new latest codex version {latest_version}."
+                if update:
+                    task = JanitorCodexUpdateTask()
+                    self.librarian_queue.put(task)
+            else:
+                level = "DEBUG"
+                log_txt = "Not fetching new latest version, not expired."
+            self.log.log(level, log_txt)
+        finally:
+            self.status_controller.finish(status)
+
+    def __init__(self, *args, **kwargs):
+        """Init Worker."""
+        self.init_worker(*args, **kwargs)

codex/librarian/bookmark/tasks.py

@@ -5,7 +5,6 @@ from dataclasses import dataclass
 from typing import Any
 
 
-@dataclass
 class BookmarkTask:
     """Bookmark Base Class."""
 
@@ -24,3 +23,14 @@ class UserActiveTask(BookmarkTask):
     """Update the user's last active status."""
 
     pk: int
+
+
+class ClearLibrarianStatusTask(BookmarkTask):
+    """Clear all librarian statuses."""
+
+
+@dataclass
+class CodexLatestVersionTask(BookmarkTask):
+    """Get the latest version."""
+
+    force: bool = False

codex/librarian/bookmark/update.py

@@ -22,7 +22,7 @@ _BOOKMARK_UPDATE_FIELDS = frozenset(
 )
 
 
-class BookmarkUpdate(GroupACLMixin):
+class BookmarkUpdateMixin(GroupACLMixin):
     """Update Bookmarks."""
 
     # Used by Bookmarkd and view.bookmark.

codex/librarian/bookmark/user_active.py

@@ -15,10 +15,9 @@ class UserActiveMixin:
     # only hit the disk to record user activity every so often
     USER_ACTIVE_RESOLUTION = timedelta(hours=1)
 
-    def __init__(self, *args, **kwargs):
+    def init_user_active(self):
         """Init the last recorded dict."""
-        super().__init__(*args, **kwargs)
-        self._user_active_recorded = {}
+        self._user_active_recorded = {}  # pyright: ignore[reportUninitializedInstanceVariable]
 
     def update_user_active(self, pk: int, log):
         """Update user active."""

codex/librarian/covers/coverd.py

@@ -1,5 +1,7 @@
 """Functions for dealing with comic cover thumbnails."""
 
+from typing_extensions import override
+
 from codex.librarian.covers.purge import CoverPurgeThread
 from codex.librarian.covers.tasks import (
     CoverCreateAllTask,
@@ -13,6 +15,7 @@ from codex.librarian.covers.tasks import (
 class CoverThread(CoverPurgeThread):
     """Create comic covers in it's own thread."""
 
+    @override
     def process_item(self, item):
         """Run the task method."""
         task = item
@@ -21,7 +24,7 @@ class CoverThread(CoverPurgeThread):
         elif isinstance(task, CoverRemoveAllTask):
             self.purge_all_comic_covers(self.librarian_queue)
         elif isinstance(task, CoverRemoveTask):
-            self.purge_comic_covers(task.pks, task.custom)
+            self.purge_comic_covers(task.pks, custom=task.custom)
         elif isinstance(task, CoverRemoveOrphansTask):
             self.cleanup_orphan_covers()
         elif isinstance(task, CoverCreateAllTask):

codex/librarian/covers/create.py

@@ -1,6 +1,8 @@
 """Create comic cover paths."""
 
+from abc import ABC
 from io import BytesIO
+from multiprocessing.queues import Queue
 from pathlib import Path
 from time import time
 
@@ -9,11 +11,11 @@ from humanize import naturaldelta
 from PIL import Image
 
 from codex.librarian.covers.path import CoverPathMixin
-from codex.librarian.covers.status import CoverStatusTypes
+from codex.librarian.covers.status import CreateCoversStatus
 from codex.librarian.covers.tasks import CoverSaveToCache
+from codex.librarian.threads import QueuedThread
 from codex.models import Comic, CustomCover
-from codex.status import Status
-from codex.threads import QueuedThread
+from codex.settings import COMICBOX_CONFIG
 
 _COVER_RATIO = 1.5372233400402415  # modal cover ratio
 THUMBNAIL_WIDTH = 165
@@ -21,7 +23,7 @@ THUMBNAIL_HEIGHT = round(THUMBNAIL_WIDTH * _COVER_RATIO)
 _THUMBNAIL_SIZE = (THUMBNAIL_WIDTH, THUMBNAIL_HEIGHT)
 
 
-class CoverCreateThread(QueuedThread, CoverPathMixin):
+class CoverCreateThread(QueuedThread, CoverPathMixin, ABC):
     """Create methods for covers."""
 
     @classmethod
@@ -40,14 +42,14 @@ class CoverCreateThread(QueuedThread, CoverPathMixin):
         return cover_thumb_buffer
 
     @classmethod
-    def _get_comic_cover_image(cls, comic_path):
+    def _get_comic_cover_image(cls, comic_path, log):
         """
         Create comic cover if none exists.
 
         Return image thumb data or path to missing file thumb.
         """
-        with Comicbox(comic_path) as cb:
-            image_data = cb.get_cover_image()
+        with Comicbox(comic_path, config=COMICBOX_CONFIG, logger=log) as car:
+            image_data = car.get_cover_page(to_pixmap=True)
         if not image_data:
             reason = "Read empty cover"
             raise ValueError(reason)
@@ -60,7 +62,9 @@ class CoverCreateThread(QueuedThread, CoverPathMixin):
             return f.read()
 
     @classmethod
-    def create_cover_from_path(cls, pk, cover_path, log, librarian_queue, custom):
+    def create_cover_from_path(
+        cls, pk: int, cover_path: str, log, librarian_queue: Queue, *, custom: bool
+    ):
         """
         Create cover for path.
 
@@ -73,7 +77,7 @@ class CoverCreateThread(QueuedThread, CoverPathMixin):
             if custom:
                 cover_image = cls._get_custom_cover_image(db_path)
             else:
-                cover_image = cls._get_comic_cover_image(db_path)
+                cover_image = cls._get_comic_cover_image(db_path, log)
             thumb_buffer = cls._create_cover_thumbnail(cover_image)
             thumb_bytes = thumb_buffer.getvalue()
             thumb_buffer.seek(0)
@@ -87,8 +91,9 @@ class CoverCreateThread(QueuedThread, CoverPathMixin):
         librarian_queue.put(task)
         return thumb_buffer
 
-    def save_cover_to_cache(self, cover_path, data):
+    def save_cover_to_cache(self, cover_path_str: str, data):
         """Save cover thumb image to the disk cache."""
+        cover_path = Path(cover_path_str)
         cover_path.parent.mkdir(exist_ok=True, parents=True)
         if data:
             with cover_path.open("wb") as cover_file:
@@ -97,12 +102,12 @@ class CoverCreateThread(QueuedThread, CoverPathMixin):
             # zero length file is code for missing.
             cover_path.touch()
 
-    def _bulk_create_comic_covers(self, pks, custom):
+    def _bulk_create_comic_covers(self, pks, custom) -> int:
         """Create bulk comic covers."""
         num_comics = len(pks)
         if not num_comics:
-            return None
-        status = Status(CoverStatusTypes.CREATE_COVERS, 0, num_comics)
+            return 0
+        status = CreateCoversStatus(0, num_comics)
         try:
             start_time = time()
             self.log.debug(f"Creating {num_comics} comic covers...")
@@ -115,27 +120,31 @@ class CoverCreateThread(QueuedThread, CoverPathMixin):
                 if cover_path.exists():
                     status.decrement_total()
                 else:
-                    # bulk contributor creates covers inline
+                    # bulk credit creates covers inline
                     data = self.create_cover_from_path(
-                        pk, cover_path, self.log, self.librarian_queue, custom=False
+                        pk,
+                        str(cover_path),
+                        self.log,
+                        self.librarian_queue,
+                        custom=False,
                     )
                     if data:
                         data.close()
                     status.increment_complete()
                 self.status_controller.update(status)
 
-            total_elapsed = naturaldelta(time() - start_time)
             desc = "custom" if custom else "comic"
-            self.log.info(
-                f"Created {status.complete} {desc} covers in {total_elapsed}."
-            )
+            count = status.complete
+            level = "INFO" if count else "DEBUG"
+            elapsed = naturaldelta(time() - start_time)
+            self.log.log(level, f"Created {count} {desc} covers in {elapsed}.")
         finally:
             self.status_controller.finish(status)
-        return status.complete
+        return status.complete or 0
 
     def create_all_covers(self):
         """Create all covers for all libraries."""
         pks = CustomCover.objects.values_list("pk", flat=True)
-        self._bulk_create_comic_covers(pks, custom=True)
+        count = self._bulk_create_comic_covers(pks, custom=True)
         pks = Comic.objects.values_list("pk", flat=True)
-        self._bulk_create_comic_covers(pks, custom=False)
+        count += self._bulk_create_comic_covers(pks, custom=False)

codex/librarian/covers/path.py

@@ -4,7 +4,7 @@ from pathlib import Path
 
 from fnvhash import fnv1a_32
 
-from codex.settings.settings import ROOT_CACHE_PATH
+from codex.settings import ROOT_CACHE_PATH
 
 
 class CoverPathMixin:

codex/librarian/covers/purge.py

@@ -2,23 +2,20 @@
 
 import os
 import shutil
+from abc import ABC
 from pathlib import Path
 
 from codex.librarian.covers.create import CoverCreateThread
-from codex.librarian.covers.status import CoverStatusTypes
+from codex.librarian.covers.status import FindOrphanCoversStatus, RemoveCoversStatus
 from codex.librarian.notifier.tasks import COVERS_CHANGED_TASK
 from codex.models import Comic
 from codex.models.paths import CustomCover
-from codex.status import Status
 
 
-class CoverPurgeThread(CoverCreateThread):
+class CoverPurgeThread(CoverCreateThread, ABC):
     """Cover Purge methods."""
 
-    _CLEANUP_STATUS_MAP = (
-        Status(CoverStatusTypes.FIND_ORPHAN),
-        Status(CoverStatusTypes.PURGE_COVERS),
-    )
+    _CLEANUP_STATUS_MAP = (FindOrphanCoversStatus, RemoveCoversStatus)
 
     @classmethod
     def _cleanup_cover_dirs(cls, path, cover_root):
@@ -31,10 +28,10 @@ class CoverPurgeThread(CoverCreateThread):
         except OSError:
             pass
 
-    def purge_cover_paths(self, cover_paths, cover_root):
+    def purge_cover_paths(self, cover_paths, cover_root) -> int:
         """Purge a set a cover paths."""
         self.log.debug(f"Removing {len(cover_paths)} possible cover thumbnails...")
-        status = Status(CoverStatusTypes.PURGE_COVERS, 0, len(cover_paths))
+        status = RemoveCoversStatus(0, len(cover_paths))
         try:
             self.status_controller.start(status)
             cover_dirs = set()
@@ -48,12 +45,11 @@ class CoverPurgeThread(CoverCreateThread):
                 self.status_controller.update(status, notify=False)
             for cover_dir in cover_dirs:
                 self._cleanup_cover_dirs(cover_dir, cover_root)
-            self.log.info(f"Removed {status.complete} cover thumbnails.")
         finally:
             self.status_controller.finish(status)
-        return status.complete
+        return status.complete or 0
 
-    def purge_comic_covers(self, pks: frozenset[int], custom: bool):
+    def purge_comic_covers(self, pks: frozenset[int], *, custom: bool):
         """Purge a set a cover paths."""
         cover_paths = self.get_cover_paths(pks, custom)
         cover_root = self.CUSTOM_COVERS_ROOT if custom else self.COVERS_ROOT
@@ -62,20 +58,26 @@ class CoverPurgeThread(CoverCreateThread):
     def purge_all_comic_covers(self, librarian_queue):
         """Purge every comic cover."""
         self.log.debug("Removing entire comic cover cache.")
+        changed = False
         try:
-            shutil.rmtree(self.COVERS_ROOT)
-            self.log.info("Removed entire comic cover cache.")
-            shutil.rmtree(self.CUSTOM_COVERS_ROOT)
-            self.log.info("Removed entire custom cover cache.")
+            if self.COVERS_ROOT.exists():
+                shutil.rmtree(self.COVERS_ROOT)
+                changed = True
+            if self.COVERS_ROOT.exists():
+                shutil.rmtree(self.CUSTOM_COVERS_ROOT)
+                changed = True
+            self.log.success("Removed entire comic cover cache and custom cover cache.")
         except OSError as exc:
             self.log.warning(exc)
-        librarian_queue.put(COVERS_CHANGED_TASK)
+        if changed:
+            librarian_queue.put(COVERS_CHANGED_TASK)
 
     def _cleanup_orphan_covers(self, cover_class, cover_root, name):
         """Remove all orphan cover thumbs."""
+        status = FindOrphanCoversStatus()
         try:
             self.log.debug(f"Removing covers from missing {name}.")
-            self.status_controller.start_many(self._CLEANUP_STATUS_MAP)
+            self.status_controller.start(status)
             pks = cover_class.objects.all().values_list("pk", flat=True)
             db_cover_paths = self.get_cover_paths(pks, custom=False)
 
@@ -87,11 +89,9 @@ class CoverPurgeThread(CoverCreateThread):
                     if fs_cover_path not in db_cover_paths:
                         orphan_cover_paths.add(fs_cover_path)
         finally:
-            self.status_controller.finish(CoverStatusTypes.FIND_ORPHAN)
+            self.status_controller.finish(status)
 
-        count = self.purge_cover_paths(orphan_cover_paths, cover_root)
-        self.log.info(f"Removed {count} covers for missing {name}.")
-        return count
+        self.purge_cover_paths(orphan_cover_paths, cover_root)
 
     def cleanup_orphan_covers(self):
         """Cleanup both comic and custom covers."""

codex/librarian/covers/status.py

@@ -1,11 +1,42 @@
 """Cover status types."""
 
-from django.db.models import TextChoices
+from abc import ABC
 
+from codex.librarian.status import Status
 
-class CoverStatusTypes(TextChoices):
-    """Cover Types."""
 
-    CREATE_COVERS = "CCC"
-    PURGE_COVERS = "CCD"
-    FIND_ORPHAN = "CFO"
+class CoversStatus(Status, ABC):
+    """Covers Status."""
+
+    ITEM_NAME = "covers"
+
+
+class CreateCoversStatus(CoversStatus):
+    """Create Covers Status."""
+
+    CODE = "CCC"
+    VERB = "Create"
+
+
+class RemoveCoversStatus(CoversStatus):
+    """Purge Covers Status."""
+
+    CODE = "CRC"
+    VERB = "Remove"
+    LOG_SUCCESS = True
+
+
+class FindOrphanCoversStatus(CoversStatus):
+    """Find Orphan Covers Status."""
+
+    CODE = "CFO"
+    ITEM_NAME = "orphan covers"
+    VERB = "Find"
+    _verbed = "Found"
+
+
+COVERS_STATII = (
+    CreateCoversStatus,
+    RemoveCoversStatus,
+    FindOrphanCoversStatus,
+)

codex/librarian/cron/crond.py

@@ -5,12 +5,13 @@ from time import sleep
 from types import MappingProxyType
 
 from django.utils import timezone as django_timezone
+from typing_extensions import override
 
-from codex.librarian.janitor.scheduled_time import get_janitor_time
-from codex.librarian.janitor.tasks import JanitorNightlyTask
+from codex.librarian.scribe.janitor.scheduled_time import get_janitor_time
+from codex.librarian.scribe.janitor.tasks import JanitorNightlyTask
 from codex.librarian.telemeter.scheduled_time import get_telemeter_time
 from codex.librarian.telemeter.tasks import TelemeterTask
-from codex.threads import NamedThread
+from codex.librarian.threads import NamedThread
 
 _TASK_TIME_FUNCTION_MAP = MappingProxyType(
     {
@@ -58,6 +59,7 @@ class CronThread(NamedThread):
                 # Times are always ordered so stop checking at the first future job.
                 break
 
+    @override
     def run(self):
         """Cron loop."""
         try: