codex 1.6.19__py3-none-any.whl → 1.7.0__py3-none-any.whl

codex/choices.py

@@ -0,0 +1,419 @@
+"""Frontend Choices, Defaults and Messages."""
+
+from collections.abc import Mapping
+from types import MappingProxyType
+
+BROWSER_BOOKMARK_FILTER_CHOICES = MappingProxyType(
+    {
+        "": "All",
+        "IN_PROGRESS": "In Progress",
+        "READ": "Read",
+        "UNREAD": "Unread",
+    },
+)
+BROWSER_ORDER_BY_CHOICES = MappingProxyType(
+    {
+        "age_rating": "Age Rating",
+        "bookmark_updated_at": "Last Read",
+        "community_rating": "Community Rating",
+        "created_at": "Added Time",
+        "critical_rating": "Critical Rating",
+        "date": "Publish Date",
+        "filename": "Filename",
+        "page_count": "Page Count",
+        "search_score": "Search Score",
+        "size": "File Size",
+        "sort_name": "Name",
+        "story_arc_number": "Story Arc Number",
+        "updated_at": "Updated Time",
+    }
+)
+_GROUP_NAMES = {
+    "p": "Publishers",
+    "i": "Imprints",
+    "s": "Series",
+    "v": "Volumes",
+}
+BROWSER_TOP_GROUP_CHOICES = MappingProxyType(
+    {
+        **_GROUP_NAMES,
+        "c": "Issues",
+        "f": "Folders",
+        "a": "Story Arcs",
+    },
+)
+VUETIFY_NULL_CODE = -1
+BROWSER_CHOICES = MappingProxyType(
+    {
+        "bookmark_filter": BROWSER_BOOKMARK_FILTER_CHOICES,
+        "order_by": BROWSER_ORDER_BY_CHOICES,
+        "top_group": BROWSER_TOP_GROUP_CHOICES,
+        "vuetify_null_code": VUETIFY_NULL_CODE,
+        "settings_group": {**_GROUP_NAMES},
+        "identifier_types": {
+            "comicvine": "Comic Vine",
+            "comixology": "Comixology",
+            "asin": "Amazon",
+            "gtin": "GTIN",
+            "isbn": "ISBN",
+            "upc": "UPC",
+        },
+    }
+)
+
+READER_CHOICES = MappingProxyType(
+    {
+        "fit_to": MappingProxyType(
+            {
+                "H": "Fit to Height",
+                "O": "Original Size",
+                "S": "Fit to Screen",
+                "W": "Fit to Width",
+            }
+        ),
+        "reading_direction": MappingProxyType(
+            {
+                "btt": "Bottom to Top",
+                "ltr": "Left to Right",
+                "rtl": "Right to Left",
+                "ttb": "Top to Bottom",
+            }
+        ),
+    }
+)
+DEFAULT_BROWSER_ROUTE = MappingProxyType({"group": "r", "pks": (0,), "page": 1})
+_DEFAULT_BROWSER_BREADCRUMBS = (DEFAULT_BROWSER_ROUTE,)
+_DEFAULT_SHOW = MappingProxyType({"i": False, "p": True, "s": True, "v": False})
+BROWSER_DEFAULTS = MappingProxyType(
+    {
+        "bookmark_filter": "",
+        "breadcrumbs": _DEFAULT_BROWSER_BREADCRUMBS,
+        "custom_covers": True,
+        "dynamic_covers": True,
+        "order_by": "sort_name",
+        "order_reverse": False,
+        "q": "",
+        # "search_results_limit": 100,
+        "show": _DEFAULT_SHOW,
+        "top_group": "p",
+        "twenty_four_hour_time": False,
+    }
+)
+READER_DEFAULTS = MappingProxyType(
+    {
+        "finish_on_last_page": True,
+        "fit_to": "S",
+        "reading_direction": "ltr",
+        "read_rtl_in_reverse": False,
+        "two_pages": False,
+    }
+)
+
+ADMIN_FLAG_CHOICES = MappingProxyType(
+    {
+        "AU": "Auto Update",
+        "FV": "Folder View",
+        "IM": "Import Metadata on Library Scan",
+        "NU": "Non Users",
+        "RG": "Registration",
+        "ST": "Send Stats",
+    }
+)
+
+ADMIN_STATUS_TITLES = MappingProxyType(
+    {
+        "CCC": "Create Covers",
+        "CCD": "Remove Covers",
+        "CFO": "Find Orphan Covers",
+        "IAF": "Adopt Orphan Folders",
+        "ICC": "Create Custom Covers",
+        "ICD": "Delete Custom Covers",
+        "ICL": "Link Custom Covers",
+        "ICM": "Custom Covers Moved",
+        "ICQ": "Discover Missing Custom Covers",
+        "ICU": "Modify Custom Covers",
+        "IDD": "Remove Folders",
+        "IDM": "Move Folders",
+        "IDU": "Modify Folders",
+        "IFC": "Create Books",
+        "IFD": "Remove Books",
+        "IFI": "Record Failed Imports",
+        "IFM": "Move Books",
+        "IFU": "Modify Books",
+        "IGU": "Update First Covers",
+        "IMC": "Link Books to Tags",
+        "IMQ": "Prepare Tags for Linking",
+        "ITC": "Create Missing Tags",
+        "ITQ": "Discover Missing Tags",
+        "ITR": "Read Book Tags",
+        "JCB": "Cleanup Orphan Bookmarks",
+        "JCD": "Cleanup Custom Covers",
+        "JCR": "Restart Codex",
+        "JCS": "Stop Codex",
+        "JCU": "Update Codex Software",
+        "JDB": "Backup Database",
+        "JDO": "Optimize Database",
+        "JFC": "Search Index Integrity Check",
+        "JFR": "Search Index Repair",
+        "JIC": "Database Integrity Check",
+        "JIF": "Database Foreign Key Check",
+        "JLV": "Check Codex Latest Version",
+        "JSD": "Cleanup Expired Sessions",
+        "JTD": "Cleanup Stale Tags",
+        "SIC": "Search Index Create",
+        "SID": "Search Index Remove Stale Records",
+        "SIO": "Search Index Optimize",
+        "SIU": "Search Index Update",
+        "SIX": "Search Index Clear",
+        "WPO": "Poll Library",
+    }
+)
+
+# Easier to store in vuetify format
+ADMIN_TASK_GROUPS = MappingProxyType(
+    {
+        "tasks": (
+            {
+                "title": "Libraries",
+                "tasks": [
+                    {
+                        "value": "poll",
+                        "title": "Poll All Libraries",
+                        "desc": "Update Libraries if changes detected",
+                    },
+                    {
+                        "value": "poll_force",
+                        "title": "Force Update All Libraries",
+                        "desc": "Forcibly update all comics in all libraries",
+                        "confirm": "This can take a long time",
+                    },
+                    {
+                        "value": "force_update_all_failed_imports",
+                        "title": "Update All Failed Imports",
+                        "desc": "Forcibly update all failed imports in all libraries",
+                        "confirm": "This can take a long time",
+                    },
+                    {
+                        "value": "watchdog_sync",
+                        "title": "Sync Watchdog with DB",
+                        "desc": "Ensure the Watchdog file watcher is enabled per database preferences for each library",
+                    },
+                ],
+            },
+            {
+                "title": "Covers",
+                "tasks": [
+                    {
+                        "value": "purge_comic_covers",
+                        "title": "Remove Comic Covers",
+                        "desc": "from every library",
+                        "confirm": "Are you sure?",
+                    },
+                    {
+                        "value": "create_all_comic_covers",
+                        "title": "Create All Comic Covers",
+                        "desc": "Pre-generate covers for every comic in every library and all custom covers",
+                        "confirm": "Are you sure?",
+                    },
+                    {
+                        "value": "force_update_groups",
+                        "title": "Update Group Timestamps",
+                        "desc": "Force the update of group timestamps. Will bust the browser cache for browser views and covers.",
+                    },
+                ],
+            },
+            {
+                "title": "Search Index",
+                "tasks": [
+                    {
+                        "value": "search_index_update",
+                        "title": "Update Search Index",
+                        "desc": "with recently changed comics",
+                    },
+                    {
+                        "value": "search_index_optimize",
+                        "title": "Optimize Search Index",
+                        "desc": "Merge Search Index for optimal lookup time. Runs nightly.",
+                    },
+                    {
+                        "value": "search_index_rebuild",
+                        "title": "Rebuild Search Index",
+                        "desc": "Delete and rebuild the search index from scratch",
+                        "confirm": "This can take a long time",
+                    },
+                    {
+                        "value": "search_index_remove_stale",
+                        "title": "Remove Stale Index Entries",
+                        "desc": "Remove search index entries that are no longer in the library.",
+                    },
+                    {
+                        "value": "search_index_abort",
+                        "title": "Abort Search Indexing",
+                        "desc": "Aborts search index update and remove tasks.",
+                    },
+                    {
+                        "value": "search_index_clear",
+                        "title": "Clear Search Index",
+                        "desc": "of all entries",
+                    },
+                    {
+                        "value": "db_fts_integrity_check",
+                        "title": "Integrity Check Search Index",
+                        "desc": "Repairs Search Index on failure. Runs nightly",
+                    },
+                    {
+                        "value": "db_fts_rebuild",
+                        "title": "Repair Search Index",
+                        "desc": "Probably faster than Rebuild if integrity check fails.",
+                    },
+                ],
+            },
+            {
+                "title": "Database",
+                "tasks": [
+                    {
+                        "value": "db_vacuum",
+                        "title": "Optimize & Compact Database",
+                        "desc": "Run the sqlite3 OPTIMIZE and VACUUM pragmas. Runs nightly",
+                    },
+                    {
+                        "value": "db_backup",
+                        "title": "Backup Database",
+                        "desc": "Runs nightly",
+                    },
+                    {
+                        "value": "db_foreign_key_check",
+                        "title": "Remove Illegal Foreign Keys",
+                        "desc": "Check for and remove illegal foreign keys. Mark affected comics for update. Runs nightly.",
+                    },
+                    {
+                        "value": "db_integrity_check",
+                        "title": "Check Database Integrity",
+                        "desc": "Check logs for results. Runs nightly.",
+                        "confirm": "Can take a while on large databases, Are you sure?",
+                    },
+                ],
+            },
+            {
+                "title": "Codex Software",
+                "tasks": [
+                    {
+                        "value": "codex_latest_version",
+                        "title": "Check for Codex Latest Version",
+                        "desc": "Check PyPi for the latest version of Codex",
+                    },
+                    {
+                        "value": "codex_update",
+                        "title": "Update Codex",
+                        "desc": "If Codex updates to a new version, it will restart",
+                        "confirm": "Are you sure?",
+                    },
+                    {
+                        "value": "codex_restart",
+                        "title": "Restart Codex Server",
+                        "desc": "Immediately",
+                        "confirm": "Are you sure?",
+                    },
+                    {
+                        "value": "codex_shutdown",
+                        "title": "Shutdown Codex Server",
+                        "desc": "Immediately",
+                        "confirm": "Are you sure?",
+                    },
+                ],
+            },
+            {
+                "title": "Notify",
+                "tasks": [
+                    {
+                        "value": "notify_library_changed",
+                        "title": "Notify Library Changed ",
+                        "desc": "Signal all clients that the libraries have changed and the browser should fetch new data.",
+                    },
+                    {
+                        "value": "notify_librarian_status",
+                        "title": "Notify Librarian Status",
+                        "desc": "Signal Admin clients to fetch librarian status.",
+                    },
+                ],
+            },
+            {
+                "title": "Cleanup",
+                "tasks": [
+                    {
+                        "value": "cleanup_fks",
+                        "title": "Remove Orphan Tags",
+                        "desc": "After deleting comics, unused linked objects remain in case new comics use them. Runs nightly.",
+                    },
+                    {
+                        "value": "cleanup_db_custom_covers",
+                        "title": "Remove Orphan Database Custom Covers",
+                        "desc": "Remove Custom Covers from the db that no longer represent custom cover images on disk. Runs nightly.",
+                    },
+                    {
+                        "value": "cleanup_sessions",
+                        "title": "Cleanup Sessions",
+                        "desc": "Remove corrupt and expired sessions. Runs nightly.",
+                    },
+                    {
+                        "value": "cleanup_covers",
+                        "title": "Remove Orphan Cover Thumbnails",
+                        "desc": "no longer have source comics or custom images. Runs nightly.",
+                    },
+                    {
+                        "value": "cleanup_bookmarks",
+                        "title": "Remove Orphan Bookmarks",
+                        "desc": "Owned by no session or user. Runs nightly.",
+                    },
+                    {
+                        "value": "adopt_folders",
+                        "title": "Adopt Orphan Folders",
+                        "desc": "Move orphaned folders from the top of the folder tree to under their correct parent. Runs nightly and at startup.",
+                    },
+                    {
+                        "value": "librarian_clear_status",
+                        "title": "Clear Librarian Statuses",
+                        "desc": "Mark all Librarian tasks finished.",
+                    },
+                    {
+                        "value": "janitor_nightly",
+                        "title": "Run Nightly Maintenance",
+                        "desc": "Runs several tasks above that also run nightly.",
+                        "confirm": "Launches several tasks that run nightly anyway.",
+                    },
+                ],
+            },
+        )
+    }
+)
+
+
+def _group_task_values(groups):
+    """Extract values into sorted tuple."""
+    values = []
+    for group in groups["tasks"]:
+        for item in group["tasks"]:
+            values.append(item["value"])
+    return tuple(sorted(values))
+
+
+ADMIN_TASKS = _group_task_values(ADMIN_TASK_GROUPS)
+WEBSOCKET_MESSAGES = MappingProxyType(
+    {
+        "messages": {
+            "FAILED_IMPORTS": "FAILED_IMPORTS",
+            "LIBRARY_CHANGED": "LIBRARY_CHANGED",
+            "LIBRARIAN_STATUS": "LIBRARIAN_STATUS",
+        }
+    }
+)
+DUMMY_NULL_NAME = "_none_"
+
+
+def mapping_to_dict(data):
+    """Convert nested Mapping objects to dicts."""
+    if isinstance(data, Mapping):
+        return {key: mapping_to_dict(value) for key, value in data.items()}
+    if isinstance(data, list | tuple | frozenset | set):
+        return [mapping_to_dict(item) for item in data]
+    return data

codex/choices_to_json.py

@@ -0,0 +1,122 @@
+#!/usr/bin/env python3
+"""Dump choices to JSON."""
+
+import json
+from collections.abc import Mapping
+from pathlib import Path
+from types import MappingProxyType
+
+from caseconverter import camelcase
+
+from codex.choices import (
+    ADMIN_FLAG_CHOICES,
+    ADMIN_STATUS_TITLES,
+    ADMIN_TASK_GROUPS,
+    BROWSER_CHOICES,
+    BROWSER_DEFAULTS,
+    READER_CHOICES,
+    READER_DEFAULTS,
+    WEBSOCKET_MESSAGES,
+)
+
+_DEFAULTS = MappingProxyType(
+    {"browser-choices.json": BROWSER_DEFAULTS, "reader-choices.json": READER_DEFAULTS}
+)
+
+_DUMPS = MappingProxyType(
+    {
+        "admin-flag-choices.json": ADMIN_FLAG_CHOICES,
+        "admin-status-titles.json": ADMIN_STATUS_TITLES,
+        "browser-choices.json": BROWSER_CHOICES,
+        "reader-choices.json": READER_CHOICES,
+    }
+)
+
+_MAP_DUMPS = MappingProxyType(
+    {
+        "admin-tasks.json": ADMIN_TASK_GROUPS,
+        "browser-defaults.json": BROWSER_DEFAULTS,
+        "browser-map.json": BROWSER_CHOICES,
+        "reader-map.json": READER_CHOICES,
+        "websocket-messages.json": WEBSOCKET_MESSAGES,
+    }
+)
+
+
+def _to_vuetify_choices(defaults, key, obj_map):
+    """Transform a dict into a list of vuetify choices."""
+    default = defaults.get(key)
+    vuetify_list = []
+    for value, title in obj_map.items():
+        vuetify_dict = {
+            "value": value,
+            "title": title,
+        }
+        if default == value:
+            vuetify_dict["default"] = True
+        vuetify_list.append(vuetify_dict)
+    return vuetify_list
+
+
+def _json_key(key):
+    """Transform key to json version."""
+    return key if key.upper() == key else camelcase(key)
+
+
+def _make_json_serializable(data):
+    """Convert nested Mapping objects to dicts."""
+    if isinstance(data, Mapping):
+        json_dict = {}
+        for key, value in data.items():
+            if key == "pks":
+                # Special route serializer
+                # XXX possibly should use actual route serializer for route not just pks
+                json_dict[key] = ",".join(str(pk) for pk in sorted(value))
+            else:
+                json_dict[_json_key(key)] = _make_json_serializable(value)
+        return json_dict
+    if isinstance(data, list | tuple | frozenset | set):
+        return [_make_json_serializable(item) for item in data]
+    return data
+
+
+def _to_vuetify_dict(fn, data):
+    """Convert mappings to vuetify dict list."""
+    vuetify_data = {}
+    defaults = _DEFAULTS.get(fn) or {}
+    for key, obj in data.items():
+        if isinstance(obj, Mapping):
+            vuetify_value = _to_vuetify_choices(defaults, key, obj)
+        else:
+            vuetify_value = _make_json_serializable(obj)
+        vuetify_data[_json_key(key)] = vuetify_value
+    return vuetify_data
+
+
+def _dump(parent_path, fn, data, vuetify=True):
+    """Dump data to json file."""
+    vuetify_data = (
+        _to_vuetify_dict(fn, data) if vuetify else _make_json_serializable(data)
+    )
+    path = parent_path / Path(fn)
+    with path.open("w") as json_file:
+        json.dump(vuetify_data, json_file, indent=2)
+        json_file.write("\n")
+
+
+def main():
+    """Dump all files."""
+    import sys
+
+    parent_path = sys.argv[1] if len(sys.argv) > 1 else "."
+    parent_path = Path(parent_path)
+    parent_path.mkdir(exist_ok=True)
+    for fn, data in _DUMPS.items():
+        _dump(parent_path, fn, data)
+
+    for fn, data in _MAP_DUMPS.items():
+        _dump(parent_path, fn, data, False)
+
+
+if __name__ == "__main__":
+    main()

codex/db.py

@@ -0,0 +1,134 @@
+"""Low level database utilities."""
+
+import sqlite3
+import subprocess
+
+from django.core.management import call_command
+from django.db import DEFAULT_DB_ALIAS, connection, connections
+from django.db.migrations.executor import MigrationExecutor
+
+from codex.librarian.janitor.integrity import (
+    cleanup_custom_cover_libraries,
+    fix_foreign_keys,
+    fts_integrity_check,
+    fts_rebuild,
+    integrity_check,
+)
+from codex.librarian.janitor.janitor import Janitor
+from codex.librarian.mp_queue import LIBRARIAN_QUEUE
+from codex.logger.logging import get_logger
+from codex.logger.mp_queue import LOG_QUEUE
+from codex.settings.settings import (
+    BACKUP_DB_PATH,
+    CONFIG_PATH,
+    DB_PATH,
+    FIX_FOREIGN_KEYS,
+    FTS_INTEGRITY_CHECK,
+    FTS_REBUILD,
+    INTEGRITY_CHECK,
+)
+from codex.version import VERSION
+
+_REPAIR_FLAG_PATH = CONFIG_PATH / "rebuild_db"
+_REBUILT_DB_PATH = DB_PATH.parent / (DB_PATH.name + ".rebuilt")
+_REPAIR_ARGS = ("sqlite3", DB_PATH, ".recover")
+
+LOG = get_logger(__name__)
+
+
+def _has_unapplied_migrations():
+    """Check if any migrations are outstanding."""
+    try:
+        connection = connections[DEFAULT_DB_ALIAS]
+        connection.prepare_database()
+        executor = MigrationExecutor(connection)
+        targets = [
+            key for key in executor.loader.graph.leaf_nodes() if key[0] == "codex"
+        ]
+        plan = executor.migration_plan(targets)
+    except Exception as exc:
+        LOG.warning(f"has_unapplied_migrations(): {exc}")
+        return False
+    else:
+        return bool(plan)
+
+
+def _get_backup_db_path(prefix):
+    suffix = f".{prefix}{BACKUP_DB_PATH.suffix}"
+    return BACKUP_DB_PATH.with_suffix(suffix)
+
+
+def _backup_db_before_migration():
+    """If there are migrations to do, backup the db."""
+    backup_path = _get_backup_db_path(f"before-v{VERSION}")
+    janitor = Janitor(LOG_QUEUE, LIBRARIAN_QUEUE)
+    janitor.backup_db(backup_path, show_status=False)
+    LOG.info("Backed up database before migrations")
+
+
+def _repair_db():
+    """Run integrity checks on startup."""
+    if FIX_FOREIGN_KEYS:
+        fix_foreign_keys()
+    if INTEGRITY_CHECK:
+        integrity_check()
+    success = fts_integrity_check() if FTS_INTEGRITY_CHECK else True
+    if FTS_REBUILD or not success:
+        fts_rebuild()
+    cleanup_custom_cover_libraries()
+
+
+def _rebuild_db():
+    """Dump and rebuild the database."""
+    # Drastic
+    if not _REPAIR_FLAG_PATH.exists():
+        return False
+
+    LOG.warning("REBUILDING DATABASE!!")
+    _REBUILT_DB_PATH.unlink(missing_ok=True)
+    recover_proc = subprocess.Popen(_REPAIR_ARGS, stdout=subprocess.PIPE)  # noqa: S603
+    with sqlite3.connect(_REBUILT_DB_PATH) as new_db_conn:
+        new_db_cur = new_db_conn.cursor()
+        if recover_proc.stdout:
+            for line in recover_proc.stdout:
+                row = line.decode().strip()
+                replaced_row = (
+                    "PRAGMA writable_schema = reset;"
+                    if row == "PRAGMA writable_schema = off;"
+                    else row
+                )
+                new_db_cur.execute(replaced_row)
+        new_db_cur.close()
+    if recover_proc.stdout:
+        recover_proc.stdout.close()
+        recover_proc.wait(timeout=15)
+
+    backup_path = _get_backup_db_path("before-rebuild")
+    DB_PATH.rename(backup_path)
+    LOG.info("Backed up old db to %s", backup_path)
+    _REBUILT_DB_PATH.replace(DB_PATH)
+    _REPAIR_FLAG_PATH.unlink(missing_ok=True)
+    LOG.info("Rebuilt database.")
+    LOG.info("You may start codex normally now.")
+    for handler in LOG.handlers:
+        handler.flush()
+    return True
+
+
+def ensure_db_schema():
+    """Ensure the db is good and up to date."""
+    LOG.info("Ensuring database is correct and up to date...")
+    table_names = connection.introspection.table_names()
+    if db_exists := "django_migrations" in table_names:
+        if _rebuild_db():
+            return False
+        _repair_db()
+
+    if not db_exists or _has_unapplied_migrations():
+        if db_exists:
+            _backup_db_before_migration()
+        call_command("migrate")
+    else:
+        LOG.info("Database up to date.")
+    LOG.info("Database ready.")
+    return True