codex 1.7.0a0__py3-none-any.whl → 1.7.0a2__py3-none-any.whl

codex/librarian/bookmark/bookmarkd.py

@@ -0,0 +1,38 @@
+"""Sends notifications to connections, reading from a queue."""
+
+from codex.librarian.bookmark.tasks import BookmarkUpdateTask
+from codex.librarian.bookmark.update import BookmarkUpdate
+from codex.threads import AggregateMessageQueuedThread
+
+
+class BookmarkThread(AggregateMessageQueuedThread, BookmarkUpdate):
+    """Aggregates Bookmark updates preventing floods updates db in batches.."""
+
+    FLOOD_DELAY = 3.0
+    MAX_DELAY = 5.0
+
+    def aggregate_items(self, item):
+        """Aggregate bookmark updates."""
+        if isinstance(item, BookmarkUpdateTask):
+            key = (tuple(item.auth_filter.items()), tuple(item.comic_filter.items()))
+            if key not in self.cache:
+                self.cache[key] = {}
+            self.cache[key].update(item.updates)
+        else:
+            self.log.warning(f"Unknown Bookmark task {item}")
+
+    def send_all_items(self):
+        """Run the task method."""
+        cleanup = set()
+        for filters, updates in self.cache.items():
+            try:
+                auth_filter, comic_filter = filters
+                auth_filter = dict(auth_filter)
+                comic_filter = dict(comic_filter)
+                self.update_bookmarks(auth_filter, comic_filter, updates)
+                self.log.debug(f"updated {auth_filter | comic_filter}")
+                cleanup.add(filters)
+            except Exception:
+                self.log.exception("Updating bookmarks")
+
+        self.cleanup_cache(cleanup)

codex/librarian/bookmark/tasks.py

@@ -0,0 +1,19 @@
+"""Bookmark Tasks."""
+
+from collections.abc import Mapping
+from dataclasses import dataclass
+from typing import Any
+
+
+@dataclass
+class BookmarkTask:
+    """Bookmark Base Class."""
+
+
+@dataclass
+class BookmarkUpdateTask(BookmarkTask):
+    """Bookmark a page."""
+
+    auth_filter: Mapping[str, int | str]
+    comic_filter: Mapping[str, Any]
+    updates: Mapping[str, Any]

codex/librarian/bookmark/update.py

@@ -0,0 +1,154 @@
+"""Sends notifications to connections, reading from a queue."""
+
+from django.contrib.auth.models import User
+from django.db.models.expressions import F
+from django.db.models.functions import Now
+from django.db.models.query import Q
+
+from codex.librarian.mp_queue import LIBRARIAN_QUEUE
+from codex.librarian.notifier.tasks import NotifierTask
+from codex.models import Bookmark, Comic
+from codex.views.auth import GroupACLMixin
+from codex.views.mixins import BookmarkSearchMixin
+
+VERTICAL_READING_DIRECTIONS = frozenset({"ttb", "btt"})
+_BOOKMARK_UPDATE_FIELDS = frozenset(
+    {
+        "page",
+        "finished",
+        "fit_to",
+        "two_pages",
+        "reading_direction",
+    }
+)
+_COMIC_ONLY_FIELDS = ("pk", "page_count")
+
+
+class BookmarkUpdate(GroupACLMixin, BookmarkSearchMixin):
+    """Update Bookmarks."""
+
+    # Used by Bookmarkd and view.bookmark.
+
+    @staticmethod
+    def _update_bookmarks_validate_page(bm, updates):
+        """Force bookmark page into valid range."""
+        page = updates.get("page")
+        if page is None:
+            return
+        max_page = bm.page_count + 1
+        page = max(min(page, max_page), 0)
+        if page == max_page:
+            # Auto finish on bookmark last page
+            bm.finished = True
+        updates["page"] = page
+
+    @staticmethod
+    def _update_bookmarks_validate_two_pages(bm, updates):
+        """Force vertical view to not use two pages."""
+        rd = updates.get("reading_direction")
+        if (
+            bm.two_pages
+            and rd in VERTICAL_READING_DIRECTIONS
+            or bm.reading_direction in VERTICAL_READING_DIRECTIONS
+        ):
+            updates["two_pages"] = None
+
+    @staticmethod
+    def _notify_library_changed(uid):
+        """Notify one user that their library changed."""
+        group = f"user_{uid}"
+        task = NotifierTask("LIBRARY_CHANGED", group)
+        LIBRARIAN_QUEUE.put(task)
+
+    @classmethod
+    def _update_bookmarks(cls, search_kwargs, updates, user) -> int:
+        """Update existing bookmarks."""
+        count = 0
+        if not updates:
+            return count
+        group_acl_filter = cls.get_group_acl_filter(Bookmark, user)
+        query_filter = group_acl_filter & Q(**search_kwargs)
+        existing_bookmarks = Bookmark.objects.filter(query_filter)
+        if updates.get("page") is not None:
+            existing_bookmarks = existing_bookmarks.annotate(
+                page_count=F("comic__page_count")
+            )
+
+        update_fields = set(updates.keys()) & _BOOKMARK_UPDATE_FIELDS
+        update_fields.add("updated_at")
+        only_fields = (*update_fields, "pk")
+
+        existing_bookmarks = existing_bookmarks.only(*only_fields)
+        update_bookmarks = []
+        for bm in existing_bookmarks:
+            cls._update_bookmarks_validate_page(bm, updates)
+            cls._update_bookmarks_validate_two_pages(bm, updates)
+            for key, value in updates.items():
+                setattr(bm, key, value)
+            bm.updated_at = Now()
+            update_bookmarks.append(bm)
+        count = len(update_bookmarks)
+        if count:
+            Bookmark.objects.bulk_update(update_bookmarks, tuple(update_fields))
+        return count
+
+    @staticmethod
+    def _create_bookmarks_validate_two_pages(updates):
+        """Force vertical view to not use two pages."""
+        if (
+            updates.get("two_pages")
+            and updates.get("reading_direction") in VERTICAL_READING_DIRECTIONS
+        ):
+            updates.pop("two_pages", None)
+
+    @classmethod
+    def _create_bookmarks(cls, comic_filter, search_kwargs, updates, user) -> int:
+        """Create new bookmarks for comics that don't exist yet."""
+        count = 0
+        cls._create_bookmarks_validate_two_pages(updates)
+        if not updates:
+            return count
+        create_bookmarks = []
+        group_acl_filter = cls.get_group_acl_filter(Comic, user)
+        exclude = {}
+        for key, value in search_kwargs.items():
+            exclude["bookmark__" + key] = value
+        query_filter = group_acl_filter & Q(**comic_filter) & ~Q(**exclude)
+        create_bookmark_comics = Comic.objects.filter(query_filter).only(
+            *_COMIC_ONLY_FIELDS
+        )
+        for comic in create_bookmark_comics:
+            defaults = {"comic": comic}
+            defaults.update(updates)
+            for key, value in search_kwargs.items():
+                if not key.startswith("comic"):
+                    defaults[key] = value
+
+            bm = Bookmark(**defaults)
+            create_bookmarks.append(bm)
+        count = len(create_bookmarks)
+        if count:
+            Bookmark.objects.bulk_create(
+                create_bookmarks,
+                update_fields=tuple(_BOOKMARK_UPDATE_FIELDS),
+                unique_fields=Bookmark._meta.unique_together[0],  # type: ignore
+            )
+        return count
+
+    @classmethod
+    def update_bookmarks(cls, auth_filter, comic_filter, updates):
+        """Update a user bookmark."""
+        search_kwargs = cls.get_bookmark_search_kwargs(auth_filter, comic_filter)
+
+        user = None
+        for key, value in auth_filter.items():
+            if key == "user":
+                user = User.objects.get(pk=value)
+            break
+
+        count = cls._update_bookmarks(search_kwargs, updates, user)
+        count += cls._create_bookmarks(comic_filter, search_kwargs, updates, user)
+        if count:
+            uid = next(iter(auth_filter.values()))
+            cls._notify_library_changed(uid)
+        return count

codex/librarian/covers/coverd.py

@@ -1,7 +1,6 @@
 """Functions for dealing with comic cover thumbnails."""
 
-from codex.librarian.covers.create import CoverCreateMixin
-from codex.librarian.covers.purge import CoverPurgeMixin
+from codex.librarian.covers.purge import CoverPurgeThread
 from codex.librarian.covers.tasks import (
     CoverCreateAllTask,
     CoverRemoveAllTask,
@@ -11,11 +10,11 @@ from codex.librarian.covers.tasks import (
 )
 
 
-class CoverThread(CoverCreateMixin, CoverPurgeMixin):
+class CoverThread(CoverPurgeThread):
     """Create comic covers in it's own thread."""
 
     def process_item(self, item):
-        """Run the contributor."""
+        """Run the task method."""
         task = item
         if isinstance(task, CoverSaveToCache):
             self.save_cover_to_cache(task.cover_path, task.data)

codex/librarian/covers/create.py

@@ -13,15 +13,16 @@ from codex.librarian.covers.status import CoverStatusTypes
 from codex.librarian.covers.tasks import CoverSaveToCache
 from codex.models import Comic, CustomCover
 from codex.status import Status
+from codex.threads import QueuedThread
 
+_COVER_RATIO = 1.5372233400402415  # modal cover ratio
+THUMBNAIL_WIDTH = 165
+THUMBNAIL_HEIGHT = round(THUMBNAIL_WIDTH * _COVER_RATIO)
+_THUMBNAIL_SIZE = (THUMBNAIL_WIDTH, THUMBNAIL_HEIGHT)
 
-class CoverCreateMixin(CoverPathMixin):
-    """Create methods for covers."""
 
-    _COVER_RATIO = 1.5372233400402415  # modal cover ratio
-    THUMBNAIL_WIDTH = 165
-    THUMBNAIL_HEIGHT = round(THUMBNAIL_WIDTH * _COVER_RATIO)
-    _THUMBNAIL_SIZE = (THUMBNAIL_WIDTH, THUMBNAIL_HEIGHT)
+class CoverCreateThread(QueuedThread, CoverPathMixin):
+    """Create methods for covers."""
 
     @classmethod
     def _create_cover_thumbnail(cls, cover_image_data):
@@ -30,7 +31,7 @@ class CoverCreateMixin(CoverPathMixin):
         with BytesIO(cover_image_data) as image_io:
             with Image.open(image_io) as cover_image:
                 cover_image.thumbnail(
-                    cls._THUMBNAIL_SIZE,
+                    _THUMBNAIL_SIZE,
                     Image.Resampling.LANCZOS,  # type: ignore
                     reducing_gap=3.0,
                 )

codex/librarian/covers/path.py

@@ -5,10 +5,9 @@ from pathlib import Path
 from fnvhash import fnv1a_32
 
 from codex.settings.settings import ROOT_CACHE_PATH
-from codex.threads import QueuedThread
 
 
-class CoverPathMixin(QueuedThread):
+class CoverPathMixin:
     """Path methods for covers."""
 
     COVERS_ROOT = ROOT_CACHE_PATH / "covers"

codex/librarian/covers/purge.py

@@ -4,7 +4,7 @@ import os
 import shutil
 from pathlib import Path
 
-from codex.librarian.covers.path import CoverPathMixin
+from codex.librarian.covers.create import CoverCreateThread
 from codex.librarian.covers.status import CoverStatusTypes
 from codex.librarian.notifier.tasks import LIBRARY_CHANGED_TASK
 from codex.models import Comic
@@ -12,7 +12,7 @@ from codex.models.paths import CustomCover
 from codex.status import Status
 
 
-class CoverPurgeMixin(CoverPathMixin):
+class CoverPurgeThread(CoverCreateThread):
     """Cover Purge methods."""
 
     _CLEANUP_STATUS_MAP = (

codex/librarian/janitor/integrity.py

@@ -35,6 +35,7 @@ def _exec_sql(sql):
     connection = connections[DEFAULT_DB_ALIAS]
     connection.prepare_database()
     with connection.cursor() as cursor:
+        cursor.execute("PRAGMA wal_checkpoint(TRUNCATE)")
         cursor.execute(sql)
         return cursor.fetchall()
 

codex/librarian/librariand.py

@@ -7,6 +7,8 @@ from typing import NamedTuple
 
 from caseconverter import snakecase
 
+from codex.librarian.bookmark.bookmarkd import BookmarkThread
+from codex.librarian.bookmark.tasks import BookmarkTask
 from codex.librarian.covers.coverd import CoverThread
 from codex.librarian.covers.tasks import CoverTask
 from codex.librarian.cron.crond import CronThread
@@ -46,6 +48,7 @@ class LibrarianDaemon(Process, LoggerBaseMixin):
     """Librarian Process."""
 
     _THREAD_CLASSES = (
+        BookmarkThread,
         NotifierThread,
         DelayedTasksThread,
         CoverThread,
@@ -88,6 +91,8 @@ class LibrarianDaemon(Process, LoggerBaseMixin):
         match task:
             case CoverTask():
                 self._threads.cover_thread.queue.put(task)
+            case BookmarkTask():
+                self._threads.bookmark_thread.queue.put(task)
             case WatchdogEventTask():
                 self._threads.watchdog_event_batcher_thread.queue.put(task)
             case ImportTask():

codex/librarian/watchdog/emitter.py

@@ -1,5 +1,6 @@
 """A Codex database event emitter for use by the observer."""
 
+from logging import DEBUG, WARNING
 from pathlib import Path
 from threading import Condition
 
@@ -91,29 +92,30 @@ class DatabasePollingEmitter(EventEmitter, WorkerBaseMixin):
 
     def _is_watch_path_ok(self, library):
         """Return a special timeout value if there's a problem with the watch dir."""
+        ok = False
+        msg = ""
+        log_level = WARNING
         if not library.poll:
             # Wait forever. Manual poll only
-            return None
-        if not self._watch_path.is_dir():
-            self.log.warning(f"Library {self._watch_path} not found.")
-            return False
-        if self._watch_path_unmounted.exists():
+            ok = None
+        elif not self._watch_path.is_dir():
+            msg = f"Library {self._watch_path} not found. Not Polling."
+        elif self._watch_path_unmounted.exists():
             # Maybe overkill of caution here
-            self.log.warning(
-                f"Library {self._watch_path} looks like an unmounted docker volume."
-            )
-            return False
-        if not tuple(self._watch_path.iterdir()):
+            msg = f"Library {self._watch_path} looks like an unmounted docker volume. Not polling."
+        elif not tuple(self._watch_path.iterdir()):
             # Maybe overkill of caution here too
-            self.log.warning(
-                f"{self._watch_path} is empty. Suspect it may be unmounted."
-            )
-            return False
-        if library.update_in_progress:
-            self.log.warning(f"Library {library.path} update in progress.")
-            return False
-
-        return True
+            msg = f"{self._watch_path} is empty. Suspect it may be unmounted. Not polling."
+        elif library.update_in_progress:
+            msg = f"Library {library.path} update in progress. Not polling."
+            log_level = DEBUG
+        else:
+            ok = True
+
+        if msg:
+            self.log.log(log_level, msg)
+
+        return ok
 
     @property
     def timeout(self) -> int | None:  # type: ignore

codex/librarian/watchdog/event_batcherd.py

@@ -20,7 +20,6 @@ from codex.threads import AggregateMessageQueuedThread
 class WatchdogEventBatcherThread(AggregateMessageQueuedThread):
     """Batch watchdog events into bulk database tasks."""
 
-    CLS_SUFFIX = -len("Event")
     DBDIFF_TASK_PARAMS = MappingProxyType(
         {
             "library_id": None,

codex/migrations/0030_nocase_collation_day_month_indexes_status_types.py

@@ -11,6 +11,10 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
+        migrations.AlterUniqueTogether(
+            name="groupauth",
+            unique_together=set(),
+        ),
         migrations.AlterField(
             model_name="adminflag",
             name="key",

codex/models/admin.py

@@ -134,8 +134,3 @@ class GroupAuth(BaseModel):
 
     group = OneToOneField(Group, db_index=True, on_delete=CASCADE)
     exclude = BooleanField(db_index=True, default=False)
-
-    class Meta(BaseModel.Meta):
-        """Constraints."""
-
-        unique_together = ("group",)

codex/models/functions.py

@@ -36,12 +36,13 @@ class FTS5Match(Lookup):
 
     def as_sql(self, compiler, connection):
         """Generate MATCH sql."""
-        lhs, lhs_params = self.process_lhs(compiler, connection)
+        # lhs, lhs_params = self.process_lhs(compiler, connection)
+        # fts_table = lhs.split(".")[0]
         rhs, rhs_params = self.process_rhs(compiler, connection)
         # MATCH works on the table itself not the one_to_one rel.
-        fts_table = lhs.split(".")[0]
-        params = lhs_params + rhs_params
-        sql = f"{fts_table} MATCH {rhs}"
+        # Force the table name without substitutions by the optimizer
+        sql = "codex_comicfts MATCH " + rhs
+        params = rhs_params
         return sql, params
 
 
@@ -62,7 +63,7 @@ class Like(Lookup):
 
 
 class ComicFTSRank(Func):
-    """Sqlite3 FTS5 inverse rank lookup."""
+    """Sqlite3 FTS5 inverse rank function."""
 
     function = "rank"
     template = '("codex_comicfts"."rank" * -1)'

codex/models/query.py

@@ -86,6 +86,12 @@ class GroupByQuerySet(QuerySet):
         obj.query.set_force_group_by(fields, model=model)
         return obj
 
+    def demote_joins(self, tables):
+        """Force INNER JOINS."""
+        obj = self._chain()  # type: ignore
+        obj.query.demote_joins(tables)
+        return obj
+
 
 class GroupByManager(Manager.from_queryset(GroupByQuerySet)):
     """Use GroupBy QuerySet."""

codex/run.py

@@ -2,6 +2,7 @@
 """The main runnable for codex. Sets up codex and runs hypercorn."""
 
 import asyncio
+import logging
 from os import execv
 
 from django.db import connection
@@ -24,14 +25,11 @@ LOG = get_logger(__name__)
 
 def codex_startup():
     """Start up codex."""
-    loggerd = CodexLogQueueListener(LOG_QUEUE)
-    loggerd.start()
-    LOG.info(f"Running Codex v{VERSION}")
-    codex_init()
-    return loggerd
+    LOG.info(f"Starting Codex v{VERSION}")
+    return codex_init()
 
 
-def database_checkpoint():
+def _database_checkpoint():
     """Write wal to disk and truncate it."""
     with connection.cursor() as cursor:
         cursor.execute("PRAGMA wal_checkpoint(TRUNCATE);")
@@ -48,8 +46,9 @@ def restart():
 
 def codex_shutdown(loggerd):
     """Shutdown for codex."""
-    database_checkpoint()
+    _database_checkpoint()
     LOG.info("Goodbye.")
+    logging.shutdown()
     loggerd.stop()
     if RESTART_EVENT.is_set():
         restart()
@@ -57,6 +56,7 @@ def codex_shutdown(loggerd):
 
 def run():
     """Run Codex."""
+    LOG.info(f"Running Codex v{VERSION}")
     librarian = LibrarianDaemon(LIBRARIAN_QUEUE, LOG_QUEUE, BROADCAST_QUEUE)
     librarian.start()
     asyncio.run(
@@ -71,8 +71,10 @@ def run():
 
 def main():
     """Set up and run Codex."""
-    loggerd = codex_startup()
-    run()
+    loggerd = CodexLogQueueListener(LOG_QUEUE)
+    loggerd.start()
+    if codex_startup():
+        run()
     codex_shutdown(loggerd)
 
 

codex/serializers/admin/tasks.py

@@ -6,11 +6,11 @@ from rest_framework.serializers import (
     Serializer,
 )
 
-from codex.serializers.choices import CHOICES
+from codex.choices import ADMIN_TASKS
 
 
 class AdminLibrarianTaskSerializer(Serializer):
     """Get tasks from front end."""
 
-    task = ChoiceField(choices=CHOICES["admin"]["tasks"])
+    task = ChoiceField(choices=ADMIN_TASKS)
     library_id = IntegerField(required=False)

codex/serializers/browser/mixins.py

@@ -1,11 +1,11 @@
 """Serializer mixins."""
 
 from datetime import datetime, timezone
+from itertools import chain
 
 from rest_framework.serializers import (
     BooleanField,
     CharField,
-    DateTimeField,
     DecimalField,
     IntegerField,
     ListField,
@@ -14,7 +14,8 @@ from rest_framework.serializers import (
 )
 
 from codex.logger.logging import get_logger
-from codex.views.const import COMIC_GROUP, EPOCH_START
+from codex.util import max_none
+from codex.views.const import EPOCH_START
 
 LOG = get_logger(__name__)
 
@@ -31,16 +32,15 @@ class BrowserAggregateSerializerMixin(Serializer):
 
     # Bookmark annotations
     page = IntegerField(read_only=True)
-    bookmark_updated_at = DateTimeField(read_only=True, allow_null=True)
     finished = BooleanField(read_only=True)
     progress = DecimalField(
         max_digits=5, decimal_places=2, read_only=True, coerce_to_string=False
     )
 
-    def get_mtime(self, obj) -> int:
-        """Compute mtime from json array aggregates."""
-        mtime = EPOCH_START
-        for dt_str in obj.updated_ats:
+    @staticmethod
+    def _get_max_updated_at(mtime, updated_ats) -> datetime:
+        """Because orm won't aggregate aggregates."""
+        for dt_str in updated_ats:
             if not dt_str:
                 continue
             try:
@@ -52,13 +52,17 @@ class BrowserAggregateSerializerMixin(Serializer):
                     f"computing group mtime: {dt_str} is not a valid datetime string."
                 )
                 continue
-
             mtime = max(dt, mtime)
+        return mtime
 
-        if obj.group != COMIC_GROUP and (
-            mbua := getattr(obj, "max_bookmark_updated_at", None)
-        ):
-            mtime = max(mtime, mbua)
-
-        # print(obj.group, obj.pk, obj.name, obj.updated_ats, obj.max_bookmark_updated_at, "max:", mtime)
+    def get_mtime(self, obj) -> int:
+        """Compute mtime from json array aggregates."""
+        updated_ats = (
+            obj.updated_ats
+            if obj.bmua_is_max
+            else chain(obj.updated_ats, obj.bookmark_updated_ats)
+        )
+        mtime = self._get_max_updated_at(EPOCH_START, updated_ats)
+        if obj.bmua_is_max:
+            mtime: datetime = max_none(mtime, obj.bookmark_updated_at, EPOCH_START)  # type: ignore
         return int(mtime.timestamp() * 1000)

codex/serializers/browser/settings.py

@@ -8,13 +8,11 @@ from rest_framework.serializers import (
     Serializer,
 )
 
+from codex.choices import BROWSER_ORDER_BY_CHOICES
 from codex.serializers.browser.filters import BrowserSettingsFilterSerializer
-from codex.serializers.choices import CHOICES, VUETIFY_NULL_CODE
 from codex.serializers.fields import BreadcrumbsField, TimestampField, TopGroupField
 from codex.serializers.route import SimpleRouteSerializer
 
-VUETIFY_NULL_CODE_STR = str(VUETIFY_NULL_CODE)
-
 
 class BrowserSettingsShowGroupFlagsSerializer(Serializer):
     """Show Group Flags."""
@@ -37,7 +35,9 @@ class BrowserCoverInputSerializerBase(BrowserFilterChoicesInputSerilalizer):
 
     custom_covers = BooleanField(required=False)
     dynamic_covers = BooleanField(required=False)
-    order_by = ChoiceField(choices=tuple(CHOICES["orderBy"].keys()), required=False)
+    order_by = ChoiceField(
+        choices=tuple(BROWSER_ORDER_BY_CHOICES.keys()), required=False
+    )
     order_reverse = BooleanField(required=False)
     show = BrowserSettingsShowGroupFlagsSerializer(required=False)