codex 1.7.0__py3-none-any.whl → 1.7.0a0__py3-none-any.whl

codex/db.py

@@ -1,7 +1,7 @@
 """Low level database utilities."""
 
+import re
 import sqlite3
-import subprocess
 
 from django.core.management import call_command
 from django.db import DEFAULT_DB_ALIAS, connection, connections
@@ -29,9 +29,9 @@ from codex.settings.settings import (
 )
 from codex.version import VERSION
 
-_REPAIR_FLAG_PATH = CONFIG_PATH / "rebuild_db"
-_REBUILT_DB_PATH = DB_PATH.parent / (DB_PATH.name + ".rebuilt")
-_REPAIR_ARGS = ("sqlite3", DB_PATH, ".recover")
+REPAIR_FLAG_PATH = CONFIG_PATH / "rebuild_db"
+DUMP_LINE_MATCHER = re.compile("TRANSACTION|ROLLBACK|COMMIT")
+REBUILT_DB_PATH = DB_PATH.parent / (DB_PATH.name + ".rebuilt")
 
 LOG = get_logger(__name__)
 
@@ -71,7 +71,8 @@ def _repair_db():
     if FIX_FOREIGN_KEYS:
         fix_foreign_keys()
     if INTEGRITY_CHECK:
-        integrity_check()
+        long = INTEGRITY_CHECK == "full"
+        integrity_check(long)
     success = fts_integrity_check() if FTS_INTEGRITY_CHECK else True
     if FTS_REBUILD or not success:
         fts_rebuild()
@@ -81,38 +82,24 @@ def _repair_db():
 def _rebuild_db():
     """Dump and rebuild the database."""
     # Drastic
-    if not _REPAIR_FLAG_PATH.exists():
-        return False
+    if not REPAIR_FLAG_PATH.exists():
+        return
 
     LOG.warning("REBUILDING DATABASE!!")
-    _REBUILT_DB_PATH.unlink(missing_ok=True)
-    recover_proc = subprocess.Popen(_REPAIR_ARGS, stdout=subprocess.PIPE)  # noqa: S603
-    with sqlite3.connect(_REBUILT_DB_PATH) as new_db_conn:
+    REPAIR_FLAG_PATH.unlink(missing_ok=True)
+    with sqlite3.connect(REBUILT_DB_PATH) as new_db_conn:
         new_db_cur = new_db_conn.cursor()
-        if recover_proc.stdout:
-            for line in recover_proc.stdout:
-                row = line.decode().strip()
-                replaced_row = (
-                    "PRAGMA writable_schema = reset;"
-                    if row == "PRAGMA writable_schema = off;"
-                    else row
-                )
-                new_db_cur.execute(replaced_row)
-        new_db_cur.close()
-    if recover_proc.stdout:
-        recover_proc.stdout.close()
-        recover_proc.wait(timeout=15)
+        with sqlite3.connect(DB_PATH) as old_db_conn:
+            for line in old_db_conn.iterdump():
+                if DUMP_LINE_MATCHER.search(line):
+                    continue
+                new_db_cur.execute(line)
 
     backup_path = _get_backup_db_path("before-rebuild")
     DB_PATH.rename(backup_path)
     LOG.info("Backed up old db to %s", backup_path)
-    _REBUILT_DB_PATH.replace(DB_PATH)
-    _REPAIR_FLAG_PATH.unlink(missing_ok=True)
+    REBUILT_DB_PATH.replace(DB_PATH)
     LOG.info("Rebuilt database.")
-    LOG.info("You may start codex normally now.")
-    for handler in LOG.handlers:
-        handler.flush()
-    return True
 
 
 def ensure_db_schema():
@@ -120,8 +107,7 @@ def ensure_db_schema():
     LOG.info("Ensuring database is correct and up to date...")
     table_names = connection.introspection.table_names()
     if db_exists := "django_migrations" in table_names:
-        if _rebuild_db():
-            return False
+        _rebuild_db()
         _repair_db()
 
     if not db_exists or _has_unapplied_migrations():
@@ -131,4 +117,3 @@ def ensure_db_schema():
     else:
         LOG.info("Database up to date.")
     LOG.info("Database ready.")
-    return True

codex/exceptions.py

@@ -11,8 +11,8 @@ from django.utils.http import urlencode
 from rest_framework.exceptions import APIException
 from rest_framework.status import HTTP_303_SEE_OTHER
 
-from codex.choices import DEFAULT_BROWSER_ROUTE
 from codex.logger.logging import get_logger
+from codex.serializers.choices import DEFAULTS
 from codex.serializers.fields import BreadcrumbsField
 from codex.serializers.route import RouteSerializer
 from codex.views.util import pop_name
@@ -24,6 +24,7 @@ if TYPE_CHECKING:
 LOG = get_logger(__name__)
 _OPDS_REDIRECT_SETTINGS_KEYS = ("order_by", "top_group")
 _REDIRECT_SETTINGS_KEYS = ("breadcrumbs", *_OPDS_REDIRECT_SETTINGS_KEYS)
+_DEFAULT_ROUTE = DEFAULTS["breadcrumbs"][0]
 
 
 class SeeOtherRedirectError(APIException):
@@ -40,7 +41,7 @@ class SeeOtherRedirectError(APIException):
 
         # Get route params
         route = detail.get("route", {})
-        params = route.get("params", DEFAULT_BROWSER_ROUTE)
+        params = route.get("params", _DEFAULT_ROUTE)
         params = pop_name(params)
         route = deepcopy(route)
         route["params"] = params

codex/librarian/covers/coverd.py

@@ -1,6 +1,7 @@
 """Functions for dealing with comic cover thumbnails."""
 
-from codex.librarian.covers.purge import CoverPurgeThread
+from codex.librarian.covers.create import CoverCreateMixin
+from codex.librarian.covers.purge import CoverPurgeMixin
 from codex.librarian.covers.tasks import (
     CoverCreateAllTask,
     CoverRemoveAllTask,
@@ -10,11 +11,11 @@ from codex.librarian.covers.tasks import (
 )
 
 
-class CoverThread(CoverPurgeThread):
+class CoverThread(CoverCreateMixin, CoverPurgeMixin):
     """Create comic covers in it's own thread."""
 
     def process_item(self, item):
-        """Run the task method."""
+        """Run the contributor."""
         task = item
         if isinstance(task, CoverSaveToCache):
             self.save_cover_to_cache(task.cover_path, task.data)

codex/librarian/covers/create.py

@@ -13,17 +13,16 @@ from codex.librarian.covers.status import CoverStatusTypes
 from codex.librarian.covers.tasks import CoverSaveToCache
 from codex.models import Comic, CustomCover
 from codex.status import Status
-from codex.threads import QueuedThread
 
-_COVER_RATIO = 1.5372233400402415  # modal cover ratio
-THUMBNAIL_WIDTH = 165
-THUMBNAIL_HEIGHT = round(THUMBNAIL_WIDTH * _COVER_RATIO)
-_THUMBNAIL_SIZE = (THUMBNAIL_WIDTH, THUMBNAIL_HEIGHT)
 
-
-class CoverCreateThread(QueuedThread, CoverPathMixin):
+class CoverCreateMixin(CoverPathMixin):
     """Create methods for covers."""
 
+    _COVER_RATIO = 1.5372233400402415  # modal cover ratio
+    THUMBNAIL_WIDTH = 165
+    THUMBNAIL_HEIGHT = round(THUMBNAIL_WIDTH * _COVER_RATIO)
+    _THUMBNAIL_SIZE = (THUMBNAIL_WIDTH, THUMBNAIL_HEIGHT)
+
     @classmethod
     def _create_cover_thumbnail(cls, cover_image_data):
         """Isolate the save thumbnail function for leak detection."""
@@ -31,7 +30,7 @@ class CoverCreateThread(QueuedThread, CoverPathMixin):
         with BytesIO(cover_image_data) as image_io:
             with Image.open(image_io) as cover_image:
                 cover_image.thumbnail(
-                    _THUMBNAIL_SIZE,
+                    cls._THUMBNAIL_SIZE,
                     Image.Resampling.LANCZOS,  # type: ignore
                     reducing_gap=3.0,
                 )

codex/librarian/covers/path.py

@@ -5,9 +5,10 @@ from pathlib import Path
 from fnvhash import fnv1a_32
 
 from codex.settings.settings import ROOT_CACHE_PATH
+from codex.threads import QueuedThread
 
 
-class CoverPathMixin:
+class CoverPathMixin(QueuedThread):
     """Path methods for covers."""
 
     COVERS_ROOT = ROOT_CACHE_PATH / "covers"

codex/librarian/covers/purge.py

@@ -4,7 +4,7 @@ import os
 import shutil
 from pathlib import Path
 
-from codex.librarian.covers.create import CoverCreateThread
+from codex.librarian.covers.path import CoverPathMixin
 from codex.librarian.covers.status import CoverStatusTypes
 from codex.librarian.notifier.tasks import LIBRARY_CHANGED_TASK
 from codex.models import Comic
@@ -12,7 +12,7 @@ from codex.models.paths import CustomCover
 from codex.status import Status
 
 
-class CoverPurgeThread(CoverCreateThread):
+class CoverPurgeMixin(CoverPathMixin):
     """Cover Purge methods."""
 
     _CLEANUP_STATUS_MAP = (

codex/librarian/janitor/integrity.py

@@ -35,7 +35,6 @@ def _exec_sql(sql):
     connection = connections[DEFAULT_DB_ALIAS]
     connection.prepare_database()
     with connection.cursor() as cursor:
-        cursor.execute("PRAGMA wal_checkpoint(TRUNCATE)")
         cursor.execute(sql)
         return cursor.fetchall()
 
@@ -277,24 +276,23 @@ def _is_integrity_ok(results):
     )
 
 
-def integrity_check(long=True, log=None):
+def integrity_check(long=False, log=None):
     """Run sqlite3 integrity check."""
-    if not log:
-        log = LOG
     pragma = "integrity_check" if long else "quick_check"
     sql = _PRAGMA_TMPL % pragma
-    log.debug(f"Running database '{sql}'...")
     results = _exec_sql(sql)
 
+    if not log:
+        log = LOG
+
     if _is_integrity_ok(results):
-        length = "" if long else "quick "
-        log.info(f"Database passed {length}integrity check.")
+        length = "thorough" if long else "quick"
+        log.info(f"Database passed {length} integrity check.")
     else:
-        log.warning(f"Database '{sql}' returned results:")
-        log.warning(results)
-        log.warning(
-            "See the README for database rebuild instructions if the above warning looks severe."
+        log.error(
+            "Database integrity compromised. See the README for database rebuild instructions."
         )
+        log.error(results)
 
 
 def fts_rebuild(log=None):
@@ -342,7 +340,7 @@ class IntegrityMixin(WorkerBaseMixin):
 
     def integrity_check(self, long=False):
         """Integrity check task."""
-        subtitle = "" if long else "Quick"
+        subtitle = "Thorough" if long else "Quick"
         status = Status(JanitorStatusTypes.INTEGRITY_CHECK, subtitle=subtitle)
         try:
             self.status_controller.start(status)

codex/librarian/janitor/tasks.py

@@ -81,7 +81,7 @@ class JanitorForeignKeyCheck(JanitorTask):
 class JanitorIntegrityCheck(JanitorTask):
     """Check integrity and warn."""
 
-    long: bool = True
+    long: bool = False
 
 
 @dataclass

codex/librarian/librariand.py

@@ -7,8 +7,6 @@ from typing import NamedTuple
 
 from caseconverter import snakecase
 
-from codex.librarian.bookmark.bookmarkd import BookmarkThread
-from codex.librarian.bookmark.tasks import BookmarkTask
 from codex.librarian.covers.coverd import CoverThread
 from codex.librarian.covers.tasks import CoverTask
 from codex.librarian.cron.crond import CronThread
@@ -48,7 +46,6 @@ class LibrarianDaemon(Process, LoggerBaseMixin):
     """Librarian Process."""
 
     _THREAD_CLASSES = (
-        BookmarkThread,
         NotifierThread,
         DelayedTasksThread,
         CoverThread,
@@ -91,8 +88,6 @@ class LibrarianDaemon(Process, LoggerBaseMixin):
         match task:
             case CoverTask():
                 self._threads.cover_thread.queue.put(task)
-            case BookmarkTask():
-                self._threads.bookmark_thread.queue.put(task)
             case WatchdogEventTask():
                 self._threads.watchdog_event_batcher_thread.queue.put(task)
             case ImportTask():

codex/librarian/watchdog/emitter.py

@@ -1,6 +1,5 @@
 """A Codex database event emitter for use by the observer."""
 
-from logging import DEBUG, WARNING
 from pathlib import Path
 from threading import Condition
 
@@ -92,30 +91,29 @@ class DatabasePollingEmitter(EventEmitter, WorkerBaseMixin):
 
     def _is_watch_path_ok(self, library):
         """Return a special timeout value if there's a problem with the watch dir."""
-        ok = False
-        msg = ""
-        log_level = WARNING
         if not library.poll:
             # Wait forever. Manual poll only
-            ok = None
-        elif not self._watch_path.is_dir():
-            msg = f"Library {self._watch_path} not found. Not Polling."
-        elif self._watch_path_unmounted.exists():
+            return None
+        if not self._watch_path.is_dir():
+            self.log.warning(f"Library {self._watch_path} not found.")
+            return False
+        if self._watch_path_unmounted.exists():
             # Maybe overkill of caution here
-            msg = f"Library {self._watch_path} looks like an unmounted docker volume. Not polling."
-        elif not tuple(self._watch_path.iterdir()):
+            self.log.warning(
+                f"Library {self._watch_path} looks like an unmounted docker volume."
+            )
+            return False
+        if not tuple(self._watch_path.iterdir()):
             # Maybe overkill of caution here too
-            msg = f"{self._watch_path} is empty. Suspect it may be unmounted. Not polling."
-        elif library.update_in_progress:
-            msg = f"Library {library.path} update in progress. Not polling."
-            log_level = DEBUG
-        else:
-            ok = True
-
-        if msg:
-            self.log.log(log_level, msg)
-
-        return ok
+            self.log.warning(
+                f"{self._watch_path} is empty. Suspect it may be unmounted."
+            )
+            return False
+        if library.update_in_progress:
+            self.log.warning(f"Library {library.path} update in progress.")
+            return False
+
+        return True
 
     @property
     def timeout(self) -> int | None:  # type: ignore

codex/librarian/watchdog/event_batcherd.py

@@ -20,6 +20,7 @@ from codex.threads import AggregateMessageQueuedThread
 class WatchdogEventBatcherThread(AggregateMessageQueuedThread):
     """Batch watchdog events into bulk database tasks."""
 
+    CLS_SUFFIX = -len("Event")
     DBDIFF_TASK_PARAMS = MappingProxyType(
         {
             "library_id": None,

codex/migrations/0030_nocase_collation_day_month_indexes_status_types.py

@@ -11,10 +11,6 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
-        migrations.AlterUniqueTogether(
-            name="groupauth",
-            unique_together=set(),
-        ),
         migrations.AlterField(
             model_name="adminflag",
             name="key",

codex/models/admin.py

@@ -134,3 +134,8 @@ class GroupAuth(BaseModel):
 
     group = OneToOneField(Group, db_index=True, on_delete=CASCADE)
     exclude = BooleanField(db_index=True, default=False)
+
+    class Meta(BaseModel.Meta):
+        """Constraints."""
+
+        unique_together = ("group",)

codex/models/functions.py

@@ -36,13 +36,12 @@ class FTS5Match(Lookup):
 
     def as_sql(self, compiler, connection):
         """Generate MATCH sql."""
-        # lhs, lhs_params = self.process_lhs(compiler, connection)
-        # fts_table = lhs.split(".")[0]
+        lhs, lhs_params = self.process_lhs(compiler, connection)
         rhs, rhs_params = self.process_rhs(compiler, connection)
         # MATCH works on the table itself not the one_to_one rel.
-        # Force the table name without substitutions by the optimizer
-        sql = "codex_comicfts MATCH " + rhs
-        params = rhs_params
+        fts_table = lhs.split(".")[0]
+        params = lhs_params + rhs_params
+        sql = f"{fts_table} MATCH {rhs}"
         return sql, params
 
 
@@ -63,7 +62,7 @@ class Like(Lookup):
 
 
 class ComicFTSRank(Func):
-    """Sqlite3 FTS5 inverse rank function."""
+    """Sqlite3 FTS5 inverse rank lookup."""
 
     function = "rank"
     template = '("codex_comicfts"."rank" * -1)'

codex/models/query.py

@@ -86,12 +86,6 @@ class GroupByQuerySet(QuerySet):
         obj.query.set_force_group_by(fields, model=model)
         return obj
 
-    def demote_joins(self, tables):
-        """Force INNER JOINS."""
-        obj = self._chain()  # type: ignore
-        obj.query.demote_joins(tables)
-        return obj
-
 
 class GroupByManager(Manager.from_queryset(GroupByQuerySet)):
     """Use GroupBy QuerySet."""

codex/run.py

@@ -2,7 +2,6 @@
 """The main runnable for codex. Sets up codex and runs hypercorn."""
 
 import asyncio
-import logging
 from os import execv
 
 from django.db import connection
@@ -25,11 +24,14 @@ LOG = get_logger(__name__)
 
 def codex_startup():
     """Start up codex."""
-    LOG.info(f"Starting Codex v{VERSION}")
-    return codex_init()
+    loggerd = CodexLogQueueListener(LOG_QUEUE)
+    loggerd.start()
+    LOG.info(f"Running Codex v{VERSION}")
+    codex_init()
+    return loggerd
 
 
-def _database_checkpoint():
+def database_checkpoint():
     """Write wal to disk and truncate it."""
     with connection.cursor() as cursor:
         cursor.execute("PRAGMA wal_checkpoint(TRUNCATE);")
@@ -46,9 +48,8 @@ def restart():
 
 def codex_shutdown(loggerd):
     """Shutdown for codex."""
-    _database_checkpoint()
+    database_checkpoint()
     LOG.info("Goodbye.")
-    logging.shutdown()
     loggerd.stop()
     if RESTART_EVENT.is_set():
         restart()
@@ -56,7 +57,6 @@ def codex_shutdown(loggerd):
 
 def run():
     """Run Codex."""
-    LOG.info(f"Running Codex v{VERSION}")
     librarian = LibrarianDaemon(LIBRARIAN_QUEUE, LOG_QUEUE, BROADCAST_QUEUE)
     librarian.start()
     asyncio.run(
@@ -71,10 +71,8 @@ def run():
 
 def main():
     """Set up and run Codex."""
-    loggerd = CodexLogQueueListener(LOG_QUEUE)
-    loggerd.start()
-    if codex_startup():
-        run()
+    loggerd = codex_startup()
+    run()
     codex_shutdown(loggerd)
 
 

codex/serializers/admin/tasks.py

@@ -6,11 +6,11 @@ from rest_framework.serializers import (
     Serializer,
 )
 
-from codex.choices import ADMIN_TASKS
+from codex.serializers.choices import CHOICES
 
 
 class AdminLibrarianTaskSerializer(Serializer):
     """Get tasks from front end."""
 
-    task = ChoiceField(choices=ADMIN_TASKS)
+    task = ChoiceField(choices=CHOICES["admin"]["tasks"])
     library_id = IntegerField(required=False)

codex/serializers/browser/mixins.py

@@ -1,11 +1,11 @@
 """Serializer mixins."""
 
 from datetime import datetime, timezone
-from itertools import chain
 
 from rest_framework.serializers import (
     BooleanField,
     CharField,
+    DateTimeField,
     DecimalField,
     IntegerField,
     ListField,
@@ -14,8 +14,7 @@ from rest_framework.serializers import (
 )
 
 from codex.logger.logging import get_logger
-from codex.util import max_none
-from codex.views.const import EPOCH_START
+from codex.views.const import COMIC_GROUP, EPOCH_START
 
 LOG = get_logger(__name__)
 
@@ -32,15 +31,16 @@ class BrowserAggregateSerializerMixin(Serializer):
 
     # Bookmark annotations
     page = IntegerField(read_only=True)
+    bookmark_updated_at = DateTimeField(read_only=True, allow_null=True)
     finished = BooleanField(read_only=True)
     progress = DecimalField(
         max_digits=5, decimal_places=2, read_only=True, coerce_to_string=False
     )
 
-    @staticmethod
-    def _get_max_updated_at(mtime, updated_ats) -> datetime:
-        """Because orm won't aggregate aggregates."""
-        for dt_str in updated_ats:
+    def get_mtime(self, obj) -> int:
+        """Compute mtime from json array aggregates."""
+        mtime = EPOCH_START
+        for dt_str in obj.updated_ats:
             if not dt_str:
                 continue
             try:
@@ -52,17 +52,13 @@ class BrowserAggregateSerializerMixin(Serializer):
                     f"computing group mtime: {dt_str} is not a valid datetime string."
                 )
                 continue
+
             mtime = max(dt, mtime)
-        return mtime
 
-    def get_mtime(self, obj) -> int:
-        """Compute mtime from json array aggregates."""
-        updated_ats = (
-            obj.updated_ats
-            if obj.bmua_is_max
-            else chain(obj.updated_ats, obj.bookmark_updated_ats)
-        )
-        mtime = self._get_max_updated_at(EPOCH_START, updated_ats)
-        if obj.bmua_is_max:
-            mtime: datetime = max_none(mtime, obj.bookmark_updated_at, EPOCH_START)  # type: ignore
+        if obj.group != COMIC_GROUP and (
+            mbua := getattr(obj, "max_bookmark_updated_at", None)
+        ):
+            mtime = max(mtime, mbua)
+
+        # print(obj.group, obj.pk, obj.name, obj.updated_ats, obj.max_bookmark_updated_at, "max:", mtime)
         return int(mtime.timestamp() * 1000)

codex/serializers/browser/settings.py

@@ -8,11 +8,13 @@ from rest_framework.serializers import (
     Serializer,
 )
 
-from codex.choices import BROWSER_ORDER_BY_CHOICES
 from codex.serializers.browser.filters import BrowserSettingsFilterSerializer
+from codex.serializers.choices import CHOICES, VUETIFY_NULL_CODE
 from codex.serializers.fields import BreadcrumbsField, TimestampField, TopGroupField
 from codex.serializers.route import SimpleRouteSerializer
 
+VUETIFY_NULL_CODE_STR = str(VUETIFY_NULL_CODE)
+
 
 class BrowserSettingsShowGroupFlagsSerializer(Serializer):
     """Show Group Flags."""
@@ -35,9 +37,7 @@ class BrowserCoverInputSerializerBase(BrowserFilterChoicesInputSerilalizer):
 
     custom_covers = BooleanField(required=False)
     dynamic_covers = BooleanField(required=False)
-    order_by = ChoiceField(
-        choices=tuple(BROWSER_ORDER_BY_CHOICES.keys()), required=False
-    )
+    order_by = ChoiceField(choices=tuple(CHOICES["orderBy"].keys()), required=False)
     order_reverse = BooleanField(required=False)
     show = BrowserSettingsShowGroupFlagsSerializer(required=False)