codex 1.7.0a0__py3-none-any.whl → 1.7.0a1__py3-none-any.whl

codex/choices.py

@@ -0,0 +1,404 @@
+"""Frontend Choices, Defaults and Messages."""
+
+from collections.abc import Mapping
+from types import MappingProxyType
+
+BROWSER_BOOKMARK_FILTER_CHOICES = MappingProxyType(
+    {
+        "": "All",
+        "IN_PROGRESS": "In Progress",
+        "READ": "Read",
+        "UNREAD": "Unread",
+    },
+)
+BROWSER_ORDER_BY_CHOICES = MappingProxyType(
+    {
+        "age_rating": "Age Rating",
+        "bookmark_updated_at": "Last Read",
+        "community_rating": "Community Rating",
+        "created_at": "Added Time",
+        "critical_rating": "Critical Rating",
+        "date": "Publish Date",
+        "filename": "Filename",
+        "page_count": "Page Count",
+        "search_score": "Search Score",
+        "size": "File Size",
+        "sort_name": "Name",
+        "story_arc_number": "Story Arc Number",
+        "updated_at": "Updated Time",
+    }
+)
+_GROUP_NAMES = {
+    "p": "Publishers",
+    "i": "Imprints",
+    "s": "Series",
+    "v": "Volumes",
+}
+BROWSER_TOP_GROUP_CHOICES = MappingProxyType(
+    {
+        **_GROUP_NAMES,
+        "c": "Issues",
+        "f": "Folders",
+        "a": "Story Arcs",
+    },
+)
+VUETIFY_NULL_CODE = -1
+BROWSER_CHOICES = MappingProxyType(
+    {
+        "bookmark_filter": BROWSER_BOOKMARK_FILTER_CHOICES,
+        "order_by": BROWSER_ORDER_BY_CHOICES,
+        "top_group": BROWSER_TOP_GROUP_CHOICES,
+        "vuetify_null_code": VUETIFY_NULL_CODE,
+        "settings_group": {**_GROUP_NAMES},
+        "identifier_types": {
+            "comicvine": "Comic Vine",
+            "comixology": "Comixology",
+            "asin": "Amazon",
+            "gtin": "GTIN",
+            "isbn": "ISBN",
+            "upc": "UPC",
+        },
+    }
+)
+
+READER_CHOICES = MappingProxyType(
+    {
+        "fit_to": MappingProxyType(
+            {
+                "H": "Fit to Height",
+                "O": "Original Size",
+                "S": "Fit to Screen",
+                "W": "Fit to Width",
+            }
+        ),
+        "reading_direction": MappingProxyType(
+            {
+                "btt": "Bottom to Top",
+                "ltr": "Left to Right",
+                "rtl": "Right to Left",
+                "ttb": "Top to Bottom",
+            }
+        ),
+    }
+)
+DEFAULT_BROWSER_ROUTE = MappingProxyType({"group": "r", "pks": (0,), "page": 1})
+_DEFAULT_BROWSER_BREADCRUMBS = (DEFAULT_BROWSER_ROUTE,)
+_DEFAULT_SHOW = MappingProxyType({"i": False, "p": True, "s": True, "v": False})
+BROWSER_DEFAULTS = MappingProxyType(
+    {
+        "bookmark_filter": "",
+        "breadcrumbs": _DEFAULT_BROWSER_BREADCRUMBS,
+        "custom_covers": True,
+        "dynamic_covers": True,
+        "order_by": "sort_name",
+        "order_reverse": False,
+        "q": "",
+        # "search_results_limit": 100,
+        "show": _DEFAULT_SHOW,
+        "top_group": "p",
+        "twenty_four_hour_time": False,
+    }
+)
+READER_DEFAULTS = MappingProxyType(
+    {
+        "finish_on_last_page": True,
+        "fit_to": "S",
+        "reading_direction": "ltr",
+        "read_rtl_in_reverse": False,
+        "two_pages": False,
+    }
+)
+
+ADMIN_FLAG_CHOICES = MappingProxyType(
+    {
+        "AU": "Auto Update",
+        "FV": "Folder View",
+        "IM": "Import Metadata on Library Scan",
+        "NU": "Non Users",
+        "RG": "Registration",
+        "ST": "Send Stats",
+    }
+)
+
+ADMIN_STATUS_TITLES = MappingProxyType(
+    {
+        "CCC": "Create Covers",
+        "CCD": "Remove Covers",
+        "CFO": "Find Orphan Covers",
+        "IAF": "Adopt Orphan Folders",
+        "ICC": "Create Custom Covers",
+        "ICD": "Delete Custom Covers",
+        "ICL": "Link Custom Covers",
+        "ICM": "Custom Covers Moved",
+        "ICQ": "Discover Missing Custom Covers",
+        "ICU": "Modify Custom Covers",
+        "IDD": "Remove Folders",
+        "IDM": "Move Folders",
+        "IDU": "Modify Folders",
+        "IFC": "Create Books",
+        "IFD": "Remove Books",
+        "IFI": "Record Failed Imports",
+        "IFM": "Move Books",
+        "IFU": "Modify Books",
+        "IGU": "Update First Covers",
+        "IMC": "Link Books to Tags",
+        "IMQ": "Prepare Tags for Linking",
+        "ITC": "Create Missing Tags",
+        "ITQ": "Discover Missing Tags",
+        "ITR": "Read Book Tags",
+        "JCB": "Cleanup Orphan Bookmarks",
+        "JCD": "Cleanup Custom Covers",
+        "JCR": "Restart Codex",
+        "JCS": "Stop Codex",
+        "JCU": "Update Codex Software",
+        "JDB": "Backup Database",
+        "JDO": "Optimize Database",
+        "JFC": "Search Index Integrity Check",
+        "JFR": "Search Index Repair",
+        "JIC": "Database Integrity Check",
+        "JIF": "Database Foreign Key Check",
+        "JLV": "Check Codex Latest Version",
+        "JSD": "Cleanup Expired Sessions",
+        "JTD": "Cleanup Stale Tags",
+        "SIC": "Search Index Create",
+        "SID": "Search Index Remove Stale Records",
+        "SIO": "Search Index Optimize",
+        "SIU": "Search Index Update",
+        "SIX": "Search Index Clear",
+        "WPO": "Poll Library",
+    }
+)
+
+# Easier to store in vuetify format
+ADMIN_TASK_GROUPS = MappingProxyType(
+    {
+        "tasks": (
+            {
+                "title": "Libraries",
+                "tasks": (
+                    {
+                        "value": "poll",
+                        "title": "Poll All Libraries",
+                        "desc": "Update Libraries if changes detected",
+                    },
+                    {
+                        "value": "poll_force",
+                        "title": "Force Update All Libraries",
+                        "desc": "Forcibly update all comics in all libraries",
+                        "confirm": "This can take a long time",
+                    },
+                    {
+                        "value": "force_update_all_failed_imports",
+                        "title": "Update All Failed Imports",
+                        "desc": "Forcibly update all failed imports in all libraries",
+                        "confirm": "This can take a long time",
+                    },
+                    {
+                        "value": "watchdog_sync",
+                        "title": "Sync Watchdog with DB",
+                        "desc": "Ensure the Watchdog file watcher is enabled per database preferences for each library",
+                    },
+                ),
+            },
+            {
+                "title": "Covers",
+                "tasks": (
+                    {
+                        "value": "purge_comic_covers",
+                        "title": "Remove Comic Covers",
+                        "desc": "from every library",
+                        "confirm": "Are you sure?",
+                    },
+                    {
+                        "value": "create_all_comic_covers",
+                        "title": "Create All Comic Covers",
+                        "desc": "Pre-generate covers for every comic in every library and all custom covers",
+                        "confirm": "Are you sure?",
+                    },
+                    {
+                        "value": "force_update_groups",
+                        "title": "Update Group Timestamps",
+                        "desc": "Force the update of group timestamps. Will bust the browser cache for browser views and covers.",
+                    },
+                ),
+            },
+            {
+                "title": "Search Index",
+                "tasks": (
+                    {
+                        "value": "search_index_update",
+                        "title": "Update Search Index",
+                        "desc": "with recently changed comics",
+                    },
+                    {
+                        "value": "search_index_rebuild",
+                        "title": "Rebuild Search Index",
+                        "desc": "Delete and rebuild the search index from scratch",
+                        "confirm": "This can take a long time",
+                    },
+                    {
+                        "value": "search_index_remove_stale",
+                        "title": "Remove Stale Index Entries",
+                        "desc": "Remove search index entries that are no longer in the library.",
+                    },
+                    {
+                        "value": "search_index_merge_small",
+                        "title": "Merge Small Search Index Segments",
+                        "desc": "Improves search lookup times. Runs nightly if the Optimize Admin Flag is off.",
+                    },
+                    {
+                        "value": "search_index_optimize",
+                        "title": "Merge Search Index Into One Segment",
+                        "desc": "Reduces disk space and improves search lookup times. Runs nightly if the Optimize Admin Flag is on.",
+                        "confirm": "This can take a long time.",
+                    },
+                    {
+                        "value": "search_index_abort",
+                        "title": "Abort Search Indexing",
+                        "desc": "Aborts search index update and remove tasks.",
+                    },
+                    {
+                        "value": "search_index_clear",
+                        "title": "Clear Search Index",
+                        "desc": "of all entries",
+                    },
+                ),
+            },
+            {
+                "title": "Database",
+                "tasks": (
+                    {
+                        "value": "db_vacuum",
+                        "title": "Optimize & Compact Database",
+                        "desc": "Run the sqlite3 OPTIMIZE and VACUUM pragmas. Runs nightly",
+                    },
+                    {
+                        "value": "db_backup",
+                        "title": "Backup Database",
+                        "desc": "Runs nightly",
+                    },
+                    {
+                        "value": "db_search_sync",
+                        "title": "Sync Search Index to DB",
+                        "desc": "Check to see if this database matches the current search index. If there is a mismatch, rebuild the search index. Runs on startup.",
+                    },
+                ),
+            },
+            {
+                "title": "Codex Software",
+                "tasks": (
+                    {
+                        "value": "codex_latest_version",
+                        "title": "Check for Codex Latest Version",
+                        "desc": "Check PyPi for the latest version of Codex",
+                    },
+                    {
+                        "value": "codex_update",
+                        "title": "Update Codex",
+                        "desc": "If Codex updates to a new version, it will restart",
+                        "confirm": "Are you sure?",
+                    },
+                    {
+                        "value": "codex_restart",
+                        "title": "Restart Codex Server",
+                        "desc": "Immediately",
+                        "confirm": "Are you sure?",
+                    },
+                    {
+                        "value": "codex_shutdown",
+                        "title": "Shutdown Codex Server",
+                        "desc": "Immediately",
+                        "confirm": "Are you sure?",
+                    },
+                ),
+            },
+            {
+                "title": "Notify",
+                "tasks": (
+                    {
+                        "value": "notify_library_changed",
+                        "title": "Notify Library Changed ",
+                        "desc": "Signal all clients that the libraries have changed and the browser should fetch new data.",
+                    },
+                    {
+                        "value": "notify_librarian_status",
+                        "title": "Notify Librarian Status",
+                        "desc": "Signal Admin clients to fetch librarian status.",
+                    },
+                ),
+            },
+            {
+                "title": "Cleanup",
+                "tasks": (
+                    {
+                        "value": "cleanup_fks",
+                        "title": "Remove Orphan Tags",
+                        "desc": "After deleting comics, unused linked objects remain in case new comics use them. Runs nightly.",
+                    },
+                    {
+                        "value": "cleanup_db_custom_covers",
+                        "title": "Remove Orphan Database Custom Covers",
+                        "desc": "Remove Custom Covers from the db that no longer represent custom cover images on disk. Runs nightly.",
+                    },
+                    {
+                        "value": "cleanup_sessions",
+                        "title": "Cleanup Sessions",
+                        "desc": "Remove corrupt and expired sessions. Runs nightly.",
+                    },
+                    {
+                        "value": "cleanup_covers",
+                        "title": "Remove Orphan Cover Thumbnails",
+                        "desc": "no longer have source comics or custom images. Runs nightly.",
+                    },
+                    {
+                        "value": "adopt_folders",
+                        "title": "Adopt Orphan Folders",
+                        "desc": "Move orphaned folders from the top of the folder tree to under their correct parent. Runs nightly and at startup.",
+                    },
+                    {
+                        "value": "librarian_clear_status",
+                        "title": "Clear Librarian Statuses",
+                        "desc": "Mark all Librarian tasks finished.",
+                    },
+                    {
+                        "value": "janitor_nightly",
+                        "title": "Run Nightly Maintenance",
+                        "desc": "Runs several tasks above that also run nightly.",
+                        "confirm": "Launches several tasks that run nightly anyway.",
+                    },
+                ),
+            },
+        )
+    }
+)
+
+
+def _group_task_values(groups):
+    """Extract values into sorted tuple."""
+    values = []
+    for group in groups["tasks"]:
+        for item in group["tasks"]:
+            values.append(item["value"])
+    return tuple(sorted(values))
+
+
+ADMIN_TASKS = _group_task_values(ADMIN_TASK_GROUPS)
+WEBSOCKET_MESSAGES = MappingProxyType(
+    {
+        "messages": {
+            "FAILED_IMPORTS": "FAILED_IMPORTS",
+            "LIBRARY_CHANGED": "LIBRARY_CHANGED",
+            "LIBRARIAN_STATUS": "LIBRARIAN_STATUS",
+        }
+    }
+)
+DUMMY_NULL_NAME = "_none_"
+
+
+def mapping_to_dict(data):
+    """Convert nested Mapping objects to dicts."""
+    if isinstance(data, Mapping):
+        return {key: mapping_to_dict(value) for key, value in data.items()}
+    if isinstance(data, list | tuple | frozenset | set):
+        return [mapping_to_dict(item) for item in data]
+    return data

codex/choices_to_json.py

@@ -0,0 +1,122 @@
+#!/usr/bin/env python3
+"""Dump choices to JSON."""
+
+import json
+from collections.abc import Mapping
+from pathlib import Path
+from types import MappingProxyType
+
+from caseconverter import camelcase
+
+from codex.choices import (
+    ADMIN_FLAG_CHOICES,
+    ADMIN_STATUS_TITLES,
+    ADMIN_TASK_GROUPS,
+    BROWSER_CHOICES,
+    BROWSER_DEFAULTS,
+    READER_CHOICES,
+    READER_DEFAULTS,
+    WEBSOCKET_MESSAGES,
+)
+
+_DEFAULTS = MappingProxyType(
+    {"browser-choices.json": BROWSER_DEFAULTS, "reader-choices.json": READER_DEFAULTS}
+)
+
+_DUMPS = MappingProxyType(
+    {
+        "admin-flag-choices.json": ADMIN_FLAG_CHOICES,
+        "admin-status-titles.json": ADMIN_STATUS_TITLES,
+        "browser-choices.json": BROWSER_CHOICES,
+        "reader-choices.json": READER_CHOICES,
+    }
+)
+
+_MAP_DUMPS = MappingProxyType(
+    {
+        "admin-tasks.json": ADMIN_TASK_GROUPS,
+        "browser-defaults.json": BROWSER_DEFAULTS,
+        "browser-map.json": BROWSER_CHOICES,
+        "reader-map.json": READER_CHOICES,
+        "websocket-messages.json": WEBSOCKET_MESSAGES,
+    }
+)
+
+
+def _to_vuetify_choices(defaults, key, obj_map):
+    """Transform a dict into a list of vuetify choices."""
+    default = defaults.get(key)
+    vuetify_list = []
+    for value, title in obj_map.items():
+        vuetify_dict = {
+            "value": value,
+            "title": title,
+        }
+        if default == value:
+            vuetify_dict["default"] = True
+        vuetify_list.append(vuetify_dict)
+    return vuetify_list
+
+
+def _json_key(key):
+    """Transform key to json version."""
+    return key if key.upper() == key else camelcase(key)
+
+
+def _make_json_serializable(data):
+    """Convert nested Mapping objects to dicts."""
+    if isinstance(data, Mapping):
+        json_dict = {}
+        for key, value in data.items():
+            if key == "pks":
+                # Special route serializer
+                # XXX possibly should use actual route serializer for route not just pks
+                json_dict[key] = ",".join(str(pk) for pk in sorted(value))
+            else:
+                json_dict[_json_key(key)] = _make_json_serializable(value)
+        return json_dict
+    if isinstance(data, list | tuple | frozenset | set):
+        return [_make_json_serializable(item) for item in data]
+    return data
+
+
+def _to_vuetify_dict(fn, data):
+    """Convert mappings to vuetify dict list."""
+    vuetify_data = {}
+    defaults = _DEFAULTS.get(fn) or {}
+    for key, obj in data.items():
+        if isinstance(obj, Mapping):
+            vuetify_value = _to_vuetify_choices(defaults, key, obj)
+        else:
+            vuetify_value = _make_json_serializable(obj)
+        vuetify_data[_json_key(key)] = vuetify_value
+    return vuetify_data
+
+
+def _dump(parent_path, fn, data, vuetify=True):
+    """Dump data to json file."""
+    vuetify_data = (
+        _to_vuetify_dict(fn, data) if vuetify else _make_json_serializable(data)
+    )
+    path = parent_path / Path(fn)
+    with path.open("w") as json_file:
+        json.dump(vuetify_data, json_file, indent=2)
+        json_file.write("\n")
+
+
+def main():
+    """Dump all files."""
+    import sys
+
+    parent_path = sys.argv[1] if len(sys.argv) > 1 else "."
+    parent_path = Path(parent_path)
+    parent_path.mkdir(exist_ok=True)
+    for fn, data in _DUMPS.items():
+        _dump(parent_path, fn, data)
+
+    for fn, data in _MAP_DUMPS.items():
+        _dump(parent_path, fn, data, False)
+
+
+if __name__ == "__main__":
+    main()

codex/db.py

@@ -1,7 +1,7 @@
 """Low level database utilities."""
 
-import re
 import sqlite3
+import subprocess
 
 from django.core.management import call_command
 from django.db import DEFAULT_DB_ALIAS, connection, connections
@@ -29,9 +29,9 @@ from codex.settings.settings import (
 )
 from codex.version import VERSION
 
-REPAIR_FLAG_PATH = CONFIG_PATH / "rebuild_db"
-DUMP_LINE_MATCHER = re.compile("TRANSACTION|ROLLBACK|COMMIT")
-REBUILT_DB_PATH = DB_PATH.parent / (DB_PATH.name + ".rebuilt")
+_REPAIR_FLAG_PATH = CONFIG_PATH / "rebuild_db"
+_REBUILT_DB_PATH = DB_PATH.parent / (DB_PATH.name + ".rebuilt")
+_REPAIR_ARGS = ("sqlite3", DB_PATH, ".recover")
 
 LOG = get_logger(__name__)
 
@@ -82,24 +82,38 @@ def _repair_db():
 def _rebuild_db():
     """Dump and rebuild the database."""
     # Drastic
-    if not REPAIR_FLAG_PATH.exists():
-        return
+    if not _REPAIR_FLAG_PATH.exists():
+        return False
 
     LOG.warning("REBUILDING DATABASE!!")
-    REPAIR_FLAG_PATH.unlink(missing_ok=True)
-    with sqlite3.connect(REBUILT_DB_PATH) as new_db_conn:
+    _REBUILT_DB_PATH.unlink(missing_ok=True)
+    recover_proc = subprocess.Popen(_REPAIR_ARGS, stdout=subprocess.PIPE)  # noqa: S603
+    with sqlite3.connect(_REBUILT_DB_PATH) as new_db_conn:
         new_db_cur = new_db_conn.cursor()
-        with sqlite3.connect(DB_PATH) as old_db_conn:
-            for line in old_db_conn.iterdump():
-                if DUMP_LINE_MATCHER.search(line):
-                    continue
-                new_db_cur.execute(line)
+        if recover_proc.stdout:
+            for line in recover_proc.stdout:
+                row = line.decode().strip()
+                replaced_row = (
+                    "PRAGMA writable_schema = reset;"
+                    if row == "PRAGMA writable_schema = off;"
+                    else row
+                )
+                new_db_cur.execute(replaced_row)
+        new_db_cur.close()
+    if recover_proc.stdout:
+        recover_proc.stdout.close()
+        recover_proc.wait(timeout=15)
 
     backup_path = _get_backup_db_path("before-rebuild")
     DB_PATH.rename(backup_path)
     LOG.info("Backed up old db to %s", backup_path)
-    REBUILT_DB_PATH.replace(DB_PATH)
+    _REBUILT_DB_PATH.replace(DB_PATH)
+    _REPAIR_FLAG_PATH.unlink(missing_ok=True)
     LOG.info("Rebuilt database.")
+    LOG.info("You may start codex normally now.")
+    for handler in LOG.handlers:
+        handler.flush()
+    return True
 
 
 def ensure_db_schema():
@@ -107,7 +121,8 @@ def ensure_db_schema():
     LOG.info("Ensuring database is correct and up to date...")
     table_names = connection.introspection.table_names()
     if db_exists := "django_migrations" in table_names:
-        _rebuild_db()
+        if _rebuild_db():
+            return False
         _repair_db()
 
     if not db_exists or _has_unapplied_migrations():
@@ -117,3 +132,4 @@ def ensure_db_schema():
     else:
         LOG.info("Database up to date.")
     LOG.info("Database ready.")
+    return True

codex/exceptions.py

@@ -11,8 +11,8 @@ from django.utils.http import urlencode
 from rest_framework.exceptions import APIException
 from rest_framework.status import HTTP_303_SEE_OTHER
 
+from codex.choices import DEFAULT_BROWSER_ROUTE
 from codex.logger.logging import get_logger
-from codex.serializers.choices import DEFAULTS
 from codex.serializers.fields import BreadcrumbsField
 from codex.serializers.route import RouteSerializer
 from codex.views.util import pop_name
@@ -24,7 +24,6 @@ if TYPE_CHECKING:
 LOG = get_logger(__name__)
 _OPDS_REDIRECT_SETTINGS_KEYS = ("order_by", "top_group")
 _REDIRECT_SETTINGS_KEYS = ("breadcrumbs", *_OPDS_REDIRECT_SETTINGS_KEYS)
-_DEFAULT_ROUTE = DEFAULTS["breadcrumbs"][0]
 
 
 class SeeOtherRedirectError(APIException):
@@ -41,7 +40,7 @@ class SeeOtherRedirectError(APIException):
 
         # Get route params
         route = detail.get("route", {})
-        params = route.get("params", _DEFAULT_ROUTE)
+        params = route.get("params", DEFAULT_BROWSER_ROUTE)
         params = pop_name(params)
         route = deepcopy(route)
         route["params"] = params

codex/librarian/bookmark/__init__.py

@@ -0,0 +1 @@
+"""Bookmark Thread."""

codex/librarian/bookmark/bookmarkd.py

@@ -0,0 +1,38 @@
+"""Sends notifications to connections, reading from a queue."""
+
+from codex.librarian.bookmark.tasks import BookmarkUpdateTask
+from codex.librarian.bookmark.update import BookmarkUpdate
+from codex.threads import AggregateMessageQueuedThread
+
+
+class BookmarkThread(AggregateMessageQueuedThread, BookmarkUpdate):
+    """Aggregates Bookmark updates preventing floods updates db in batches.."""
+
+    FLOOD_DELAY = 3.0
+    MAX_DELAY = 5.0
+
+    def aggregate_items(self, item):
+        """Aggregate bookmark updates."""
+        if isinstance(item, BookmarkUpdateTask):
+            key = (tuple(item.auth_filter.items()), tuple(item.comic_filter.items()))
+            if key not in self.cache:
+                self.cache[key] = {}
+            self.cache[key].update(item.updates)
+        else:
+            self.log.warning(f"Unknown Bookmark task {item}")
+
+    def send_all_items(self):
+        """Run the task method."""
+        cleanup = set()
+        for filters, updates in self.cache.items():
+            try:
+                auth_filter, comic_filter = filters
+                auth_filter = dict(auth_filter)
+                comic_filter = dict(comic_filter)
+                self.update_bookmarks(auth_filter, comic_filter, updates)
+                self.log.debug(f"updated {auth_filter | comic_filter}")
+                cleanup.add(filters)
+            except Exception:
+                self.log.exception("Updating bookmarks")
+
+        self.cleanup_cache(cleanup)