codex 1.6.19__py3-none-any.whl → 1.7.0__py3-none-any.whl

codex/exceptions.py

@@ -11,8 +11,8 @@ from django.utils.http import urlencode
 from rest_framework.exceptions import APIException
 from rest_framework.status import HTTP_303_SEE_OTHER
 
+from codex.choices import DEFAULT_BROWSER_ROUTE
 from codex.logger.logging import get_logger
-from codex.serializers.choices import DEFAULTS
 from codex.serializers.fields import BreadcrumbsField
 from codex.serializers.route import RouteSerializer
 from codex.views.util import pop_name
@@ -24,7 +24,6 @@ if TYPE_CHECKING:
 LOG = get_logger(__name__)
 _OPDS_REDIRECT_SETTINGS_KEYS = ("order_by", "top_group")
 _REDIRECT_SETTINGS_KEYS = ("breadcrumbs", *_OPDS_REDIRECT_SETTINGS_KEYS)
-_DEFAULT_ROUTE = DEFAULTS["breadcrumbs"][0]
 
 
 class SeeOtherRedirectError(APIException):
@@ -41,7 +40,7 @@ class SeeOtherRedirectError(APIException):
 
         # Get route params
         route = detail.get("route", {})
-        params = route.get("params", _DEFAULT_ROUTE)
+        params = route.get("params", DEFAULT_BROWSER_ROUTE)
         params = pop_name(params)
         route = deepcopy(route)
         route["params"] = params

codex/librarian/bookmark/__init__.py

@@ -0,0 +1 @@
+"""Bookmark Thread."""

codex/librarian/bookmark/bookmarkd.py

@@ -0,0 +1,38 @@
+"""Sends notifications to connections, reading from a queue."""
+
+from codex.librarian.bookmark.tasks import BookmarkUpdateTask
+from codex.librarian.bookmark.update import BookmarkUpdate
+from codex.threads import AggregateMessageQueuedThread
+
+
+class BookmarkThread(AggregateMessageQueuedThread, BookmarkUpdate):
+    """Aggregates Bookmark updates preventing floods updates db in batches.."""
+
+    FLOOD_DELAY = 3.0
+    MAX_DELAY = 5.0
+
+    def aggregate_items(self, item):
+        """Aggregate bookmark updates."""
+        if isinstance(item, BookmarkUpdateTask):
+            key = (tuple(item.auth_filter.items()), tuple(item.comic_filter.items()))
+            if key not in self.cache:
+                self.cache[key] = {}
+            self.cache[key].update(item.updates)
+        else:
+            self.log.warning(f"Unknown Bookmark task {item}")
+
+    def send_all_items(self):
+        """Run the task method."""
+        cleanup = set()
+        for filters, updates in self.cache.items():
+            try:
+                auth_filter, comic_filter = filters
+                auth_filter = dict(auth_filter)
+                comic_filter = dict(comic_filter)
+                self.update_bookmarks(auth_filter, comic_filter, updates)
+                self.log.debug(f"updated {auth_filter | comic_filter}")
+                cleanup.add(filters)
+            except Exception:
+                self.log.exception("Updating bookmarks")
+
+        self.cleanup_cache(cleanup)

codex/librarian/bookmark/tasks.py

@@ -0,0 +1,19 @@
+"""Bookmark Tasks."""
+
+from collections.abc import Mapping
+from dataclasses import dataclass
+from typing import Any
+
+
+@dataclass
+class BookmarkTask:
+    """Bookmark Base Class."""
+
+
+@dataclass
+class BookmarkUpdateTask(BookmarkTask):
+    """Bookmark a page."""
+
+    auth_filter: Mapping[str, int | str]
+    comic_filter: Mapping[str, Any]
+    updates: Mapping[str, Any]

codex/librarian/bookmark/update.py

@@ -0,0 +1,154 @@
+"""Sends notifications to connections, reading from a queue."""
+
+from django.contrib.auth.models import User
+from django.db.models.expressions import F
+from django.db.models.functions import Now
+from django.db.models.query import Q
+
+from codex.librarian.mp_queue import LIBRARIAN_QUEUE
+from codex.librarian.notifier.tasks import NotifierTask
+from codex.models import Bookmark, Comic
+from codex.views.auth import GroupACLMixin
+from codex.views.mixins import BookmarkSearchMixin
+
+VERTICAL_READING_DIRECTIONS = frozenset({"ttb", "btt"})
+_BOOKMARK_UPDATE_FIELDS = frozenset(
+    {
+        "page",
+        "finished",
+        "fit_to",
+        "two_pages",
+        "reading_direction",
+    }
+)
+_COMIC_ONLY_FIELDS = ("pk", "page_count")
+
+
+class BookmarkUpdate(GroupACLMixin, BookmarkSearchMixin):
+    """Update Bookmarks."""
+
+    # Used by Bookmarkd and view.bookmark.
+
+    @staticmethod
+    def _update_bookmarks_validate_page(bm, updates):
+        """Force bookmark page into valid range."""
+        page = updates.get("page")
+        if page is None:
+            return
+        max_page = bm.page_count + 1
+        page = max(min(page, max_page), 0)
+        if page == max_page:
+            # Auto finish on bookmark last page
+            bm.finished = True
+        updates["page"] = page
+
+    @staticmethod
+    def _update_bookmarks_validate_two_pages(bm, updates):
+        """Force vertical view to not use two pages."""
+        rd = updates.get("reading_direction")
+        if (
+            bm.two_pages
+            and rd in VERTICAL_READING_DIRECTIONS
+            or bm.reading_direction in VERTICAL_READING_DIRECTIONS
+        ):
+            updates["two_pages"] = None
+
+    @staticmethod
+    def _notify_library_changed(uid):
+        """Notify one user that their library changed."""
+        group = f"user_{uid}"
+        task = NotifierTask("LIBRARY_CHANGED", group)
+        LIBRARIAN_QUEUE.put(task)
+
+    @classmethod
+    def _update_bookmarks(cls, search_kwargs, updates, user) -> int:
+        """Update existing bookmarks."""
+        count = 0
+        if not updates:
+            return count
+        group_acl_filter = cls.get_group_acl_filter(Bookmark, user)
+        query_filter = group_acl_filter & Q(**search_kwargs)
+        existing_bookmarks = Bookmark.objects.filter(query_filter)
+        if updates.get("page") is not None:
+            existing_bookmarks = existing_bookmarks.annotate(
+                page_count=F("comic__page_count")
+            )
+
+        update_fields = set(updates.keys()) & _BOOKMARK_UPDATE_FIELDS
+        update_fields.add("updated_at")
+        only_fields = (*update_fields, "pk")
+
+        existing_bookmarks = existing_bookmarks.only(*only_fields)
+        update_bookmarks = []
+        for bm in existing_bookmarks:
+            cls._update_bookmarks_validate_page(bm, updates)
+            cls._update_bookmarks_validate_two_pages(bm, updates)
+            for key, value in updates.items():
+                setattr(bm, key, value)
+            bm.updated_at = Now()
+            update_bookmarks.append(bm)
+        count = len(update_bookmarks)
+        if count:
+            Bookmark.objects.bulk_update(update_bookmarks, tuple(update_fields))
+        return count
+
+    @staticmethod
+    def _create_bookmarks_validate_two_pages(updates):
+        """Force vertical view to not use two pages."""
+        if (
+            updates.get("two_pages")
+            and updates.get("reading_direction") in VERTICAL_READING_DIRECTIONS
+        ):
+            updates.pop("two_pages", None)
+
+    @classmethod
+    def _create_bookmarks(cls, comic_filter, search_kwargs, updates, user) -> int:
+        """Create new bookmarks for comics that don't exist yet."""
+        count = 0
+        cls._create_bookmarks_validate_two_pages(updates)
+        if not updates:
+            return count
+        create_bookmarks = []
+        group_acl_filter = cls.get_group_acl_filter(Comic, user)
+        exclude = {}
+        for key, value in search_kwargs.items():
+            exclude["bookmark__" + key] = value
+        query_filter = group_acl_filter & Q(**comic_filter) & ~Q(**exclude)
+        create_bookmark_comics = Comic.objects.filter(query_filter).only(
+            *_COMIC_ONLY_FIELDS
+        )
+        for comic in create_bookmark_comics:
+            defaults = {"comic": comic}
+            defaults.update(updates)
+            for key, value in search_kwargs.items():
+                if not key.startswith("comic"):
+                    defaults[key] = value
+
+            bm = Bookmark(**defaults)
+            create_bookmarks.append(bm)
+        count = len(create_bookmarks)
+        if count:
+            Bookmark.objects.bulk_create(
+                create_bookmarks,
+                update_fields=tuple(_BOOKMARK_UPDATE_FIELDS),
+                unique_fields=Bookmark._meta.unique_together[0],  # type: ignore
+            )
+        return count
+
+    @classmethod
+    def update_bookmarks(cls, auth_filter, comic_filter, updates):
+        """Update a user bookmark."""
+        search_kwargs = cls.get_bookmark_search_kwargs(auth_filter, comic_filter)
+
+        user = None
+        for key, value in auth_filter.items():
+            if key == "user":
+                user = User.objects.get(pk=value)
+            break
+
+        count = cls._update_bookmarks(search_kwargs, updates, user)
+        count += cls._create_bookmarks(comic_filter, search_kwargs, updates, user)
+        if count:
+            uid = next(iter(auth_filter.values()))
+            cls._notify_library_changed(uid)
+        return count

codex/librarian/covers/coverd.py

@@ -1,7 +1,6 @@
 """Functions for dealing with comic cover thumbnails."""
 
-from codex.librarian.covers.create import CoverCreateMixin
-from codex.librarian.covers.purge import CoverPurgeMixin
+from codex.librarian.covers.purge import CoverPurgeThread
 from codex.librarian.covers.tasks import (
     CoverCreateAllTask,
     CoverRemoveAllTask,
@@ -11,11 +10,11 @@ from codex.librarian.covers.tasks import (
 )
 
 
-class CoverThread(CoverCreateMixin, CoverPurgeMixin):
+class CoverThread(CoverPurgeThread):
     """Create comic covers in it's own thread."""
 
     def process_item(self, item):
-        """Run the contributor."""
+        """Run the task method."""
         task = item
         if isinstance(task, CoverSaveToCache):
             self.save_cover_to_cache(task.cover_path, task.data)

codex/librarian/covers/create.py

@@ -13,15 +13,16 @@ from codex.librarian.covers.status import CoverStatusTypes
 from codex.librarian.covers.tasks import CoverSaveToCache
 from codex.models import Comic, CustomCover
 from codex.status import Status
+from codex.threads import QueuedThread
 
+_COVER_RATIO = 1.5372233400402415  # modal cover ratio
+THUMBNAIL_WIDTH = 165
+THUMBNAIL_HEIGHT = round(THUMBNAIL_WIDTH * _COVER_RATIO)
+_THUMBNAIL_SIZE = (THUMBNAIL_WIDTH, THUMBNAIL_HEIGHT)
 
-class CoverCreateMixin(CoverPathMixin):
-    """Create methods for covers."""
 
-    _COVER_RATIO = 1.5372233400402415  # modal cover ratio
-    THUMBNAIL_WIDTH = 165
-    THUMBNAIL_HEIGHT = round(THUMBNAIL_WIDTH * _COVER_RATIO)
-    _THUMBNAIL_SIZE = (THUMBNAIL_WIDTH, THUMBNAIL_HEIGHT)
+class CoverCreateThread(QueuedThread, CoverPathMixin):
+    """Create methods for covers."""
 
     @classmethod
     def _create_cover_thumbnail(cls, cover_image_data):
@@ -30,7 +31,7 @@ class CoverCreateMixin(CoverPathMixin):
         with BytesIO(cover_image_data) as image_io:
             with Image.open(image_io) as cover_image:
                 cover_image.thumbnail(
-                    cls._THUMBNAIL_SIZE,
+                    _THUMBNAIL_SIZE,
                     Image.Resampling.LANCZOS,  # type: ignore
                     reducing_gap=3.0,
                 )

codex/librarian/covers/path.py

@@ -5,10 +5,9 @@ from pathlib import Path
 from fnvhash import fnv1a_32
 
 from codex.settings.settings import ROOT_CACHE_PATH
-from codex.threads import QueuedThread
 
 
-class CoverPathMixin(QueuedThread):
+class CoverPathMixin:
     """Path methods for covers."""
 
     COVERS_ROOT = ROOT_CACHE_PATH / "covers"

codex/librarian/covers/purge.py

@@ -4,7 +4,7 @@ import os
 import shutil
 from pathlib import Path
 
-from codex.librarian.covers.path import CoverPathMixin
+from codex.librarian.covers.create import CoverCreateThread
 from codex.librarian.covers.status import CoverStatusTypes
 from codex.librarian.notifier.tasks import LIBRARY_CHANGED_TASK
 from codex.models import Comic
@@ -12,7 +12,7 @@ from codex.models.paths import CustomCover
 from codex.status import Status
 
 
-class CoverPurgeMixin(CoverPathMixin):
+class CoverPurgeThread(CoverCreateThread):
     """Cover Purge methods."""
 
     _CLEANUP_STATUS_MAP = (
@@ -45,7 +45,7 @@ class CoverPurgeMixin(CoverPathMixin):
                 except FileNotFoundError:
                     status.decrement_total()
                 cover_dirs.add(cover_path.parent)
-                self.status_controller.update(status)
+                self.status_controller.update(status, notify=False)
             for cover_dir in cover_dirs:
                 self._cleanup_cover_dirs(cover_dir, cover_root)
             self.log.info(f"Removed {status.complete} cover thumbnails.")

codex/librarian/importer/const.py

@@ -115,7 +115,13 @@ DICT_MODEL_REL_LINK_MAP = MappingProxyType(
 # BULK UPDATE #
 ###############
 
-_EXCLUDEBULK_UPDATE_COMIC_FIELDS = {"bookmark", "created_at", "id", "library"}
+_EXCLUDEBULK_UPDATE_COMIC_FIELDS = {
+    "bookmark",
+    "created_at",
+    "id",
+    "library",
+    "comicfts",
+}
 GROUP_BASE_FIELDS = ("name", "sort_name")
 BULK_UPDATE_COMIC_FIELDS = tuple(
     sorted(

codex/librarian/importer/create_fks.py

@@ -213,8 +213,11 @@ class CreateForeignKeysImporter(CreateCoversImporter):
         # create each depth level first to ensure we can assign parents
         for depth_level in sorted(folder_path_dict):
             paths = sorted(folder_path_dict[depth_level])
-            count += self._bulk_folders_create_depth_level(paths, status)
-            self.log.debug(f"Created {count} Folders at depth level {depth_level}")
+            level_count = self._bulk_folders_create_depth_level(paths, status)
+            count += level_count
+            self.log.debug(
+                f"Created {level_count} Folders at depth level {depth_level}"
+            )
 
         if count:
             self.log.info(f"Created {count} Folders.")

codex/librarian/importer/importerd.py

@@ -12,6 +12,7 @@ from codex.librarian.importer.tasks import (
     LazyImportComicsTask,
     UpdateGroupsTask,
 )
+from codex.librarian.janitor.tasks import JanitorAdoptOrphanFoldersFinishedTask
 from codex.librarian.notifier.tasks import LIBRARY_CHANGED_TASK
 from codex.models import Comic, Folder, Library
 from codex.status import Status
@@ -88,20 +89,24 @@ class ComicImporterThread(QueuedThread):
         importer.bulk_folders_moved()
         return True
 
-    def _adopt_orphan_folders(self):
+    def _adopt_orphan_folders(self, janitor=False):
         """Find orphan folders and move them into their correct place."""
         status = Status(ImportStatusTypes.ADOPT_FOLDERS)
         moved_status = Status(ImportStatusTypes.DIRS_MOVED)
-        self.status_controller.start(status)
-        self.status_controller.start(moved_status)
-        libraries = Library.objects.filter(covers_only=False).only("path")
-        for library in libraries.iterator():
-            folders_left = True
-            while folders_left:
-                # Run until there are no orphan folders
-                folders_left = self._adopt_orphan_folders_for_library(library)
-
-        self.status_controller.finish_many((moved_status, status))
+        try:
+            self.status_controller.start(status)
+            self.status_controller.start(moved_status)
+            libraries = Library.objects.filter(covers_only=False).only("path")
+            for library in libraries.iterator():
+                folders_left = True
+                while folders_left:
+                    # Run until there are no orphan folders
+                    folders_left = self._adopt_orphan_folders_for_library(library)
+        finally:
+            self.status_controller.finish_many((moved_status, status))
+            if janitor:
+                next_task = JanitorAdoptOrphanFoldersFinishedTask()
+                self.librarian_queue.put(next_task)
 
     def process_item(self, item):
         """Run the updater."""
@@ -113,6 +118,6 @@ class ComicImporterThread(QueuedThread):
         elif isinstance(task, UpdateGroupsTask):
             self._update_groups(task)
         elif isinstance(task, AdoptOrphanFoldersTask):
-            self._adopt_orphan_folders()
+            self._adopt_orphan_folders(task.janitor)
         else:
             self.log.warning(f"Bad task sent to library updater {task}")

codex/librarian/importer/moved.py

@@ -164,17 +164,23 @@ class MovedImporter(AggregateMetadataImporter):
         self.status_controller.finish(status)
 
     def _bulk_move_folders(
-        self, src_folder_paths, dest_parent_folders_map, dirs_moved: bidict, status
+        self,
+        src_folder_paths_with_existing_dest_parents,
+        dest_parent_folders_map,
+        dirs_moved: bidict[str, str],
+        status,
     ):
         """Bulk move folders."""
+        # Move collisions removed before this
         folders_to_move = (
-            Folder.objects.filter(library=self.library, path__in=src_folder_paths)
+            Folder.objects.filter(
+                library=self.library,
+                path__in=src_folder_paths_with_existing_dest_parents,
+            )
             .only(*MOVED_BULK_FOLDER_UPDATE_FIELDS)
             .order_by("path")
         )
 
-        # TODO what if a folder with that path already exists in the db?
-
         new_paths = set()
         update_folders = []
         for folder in folders_to_move.iterator():
@@ -201,7 +207,7 @@ class MovedImporter(AggregateMetadataImporter):
         self.status_controller.update(status, notify=False)
 
     def _bulk_move_folders_under_existing_parents(
-        self, dest_parent_folder_paths_map, dirs_moved, status
+        self, dest_parent_folder_paths_map, dirs_moved: bidict, status
     ):
         """Move folders under existing folders."""
         while True:
@@ -214,25 +220,32 @@ class MovedImporter(AggregateMetadataImporter):
 
             # Create a list of folders than can be moved under existing folders.
             dest_parent_folders_map = {}
-            src_folder_paths = []
+            src_folder_paths_with_existing_dest_parents = []
             for extant_parent_folder in extant_parent_folders:
                 dest_parent_folders_map[extant_parent_folder.path] = (
                     extant_parent_folder
                 )
                 if dest_folder_paths := dest_parent_folder_paths_map.pop(
-                    extant_parent_folder.path
+                    extant_parent_folder.path, None
                 ):
                     for dest_folder_path in dest_folder_paths:
                         src_path = dirs_moved.inverse[dest_folder_path]
-                        src_folder_paths.append(src_path)
+                        src_folder_paths_with_existing_dest_parents.append(src_path)
 
-            if not src_folder_paths:
+            if not src_folder_paths_with_existing_dest_parents:
                 return
-            src_folder_paths = sorted(src_folder_paths)
-            self.log.debug(f"Moving folders under existing parents: {src_folder_paths}")
+            src_folder_paths_with_existing_dest_parents = sorted(
+                src_folder_paths_with_existing_dest_parents
+            )
+            self.log.debug(
+                f"Moving folders under existing parents: {src_folder_paths_with_existing_dest_parents}"
+            )
 
             self._bulk_move_folders(
-                src_folder_paths, dest_parent_folders_map, dirs_moved, status
+                src_folder_paths_with_existing_dest_parents,
+                dest_parent_folders_map,
+                dirs_moved,
+                status,
             )
 
     def _get_move_create_folders_one_layer(self, dest_parent_folder_paths_map):
@@ -260,12 +273,30 @@ class MovedImporter(AggregateMetadataImporter):
 
         return frozenset(create_folder_paths_one_layer)
 
+    def _remove_move_collisions(self, dirs_moved: bidict[str, str]):
+        """Remove moves that would collide with an existing Folder."""
+        dest_paths = set(dirs_moved.values())
+        collision_dest_paths = Folder.objects.filter(
+            library=self.library, path__in=dest_paths
+        ).values_list("path", flat=True)
+        if not collision_dest_paths:
+            return
+        collision_dest_paths = sorted(set(collision_dest_paths))
+        for collision_dest_path in collision_dest_paths:
+            dirs_moved.inverse.pop(collision_dest_path, None)
+        self.log.warning(
+            f"Not moving folders to destinations that would collide with existing database folders: {collision_dest_paths}"
+        )
+
     def _bulk_move_folders_and_create_parents(self, status):
         """Find folders that can be moved without creating parents."""
         dirs_moved = bidict(self.task.dirs_moved)
 
+        self._remove_move_collisions(dirs_moved)
+
+        dest_paths = sorted(set(dirs_moved.values()))
         dest_parent_folder_paths_map = {}
-        for dest_path in sorted(set(dirs_moved.values())):
+        for dest_path in dest_paths:
             parent = str(Path(dest_path).parent)
             if parent not in dest_parent_folder_paths_map:
                 dest_parent_folder_paths_map[parent] = set()

codex/librarian/importer/tasks.py

@@ -45,6 +45,8 @@ class LazyImportComicsTask(ImportTask):
 class AdoptOrphanFoldersTask(ImportTask):
     """Move orphaned folders into a correct tree position."""
 
+    janitor: bool = False
+
 
 @dataclass
 class UpdateGroupsTask(ImportTask):

codex/librarian/janitor/cleanup.py

@@ -35,6 +35,7 @@ from codex.models import (
     Team,
     Volume,
 )
+from codex.models.bookmark import Bookmark
 from codex.models.paths import CustomCover
 from codex.status import Status
 from codex.worker_base import WorkerBaseMixin
@@ -84,15 +85,15 @@ CLEANUP_MAP = MappingProxyType(
 class CleanupMixin(WorkerBaseMixin):
     """Cleanup methods for Janitor."""
 
-    def _bulk_cleanup_fks(self, classes, field_name, status):
+    def _bulk_cleanup_fks(self, models, field_name, status):
         """Remove foreign keys that aren't used anymore."""
-        for cls in classes:
+        for model in models:
             filter_dict = {f"{field_name}__isnull": True}
-            query = cls.objects.filter(**filter_dict)
-            count = query.count()
-            query.delete()
+            qs = model.objects.filter(**filter_dict)
+            count = qs.count()
+            qs.delete()
             if count:
-                self.log.info(f"Deleted {count} orphan {cls.__name__}s")
+                self.log.info(f"Deleted {count} orphan {model.__name__}s")
             status.complete += count
             self.status_controller.update(status)
 
@@ -102,8 +103,8 @@ class CleanupMixin(WorkerBaseMixin):
         try:
             self.status_controller.start(status)
             self.log.debug("Cleaning up unused foreign keys...")
-            for field_name, classes in CLEANUP_MAP.items():
-                self._bulk_cleanup_fks(classes, field_name, status)
+            for field_name, models in CLEANUP_MAP.items():
+                self._bulk_cleanup_fks(models, field_name, status)
             level = logging.INFO if status.complete else logging.DEBUG
             self.log.log(level, f"Cleaned up {status.complete} unused foreign keys.")
         finally:
@@ -152,3 +153,15 @@ class CleanupMixin(WorkerBaseMixin):
         finally:
             until = time() + 1
             self.status_controller.finish(status, until=until)
+
+    def cleanup_orphan_bookmarks(self):
+        """Delete bookmarks without users or sessions."""
+        status = Status(JanitorStatusTypes.CLEANUP_BOOKMARKS)
+        try:
+            self.status_controller.start(status)
+            orphan_bms = Bookmark.objects.filter(session=None, user=None)
+            if count := orphan_bms.count():
+                orphan_bms.delete()
+                self.log.info(f"Deleted {count} orphan bookmarks.")
+        finally:
+            self.status_controller.finish(status)