codex 1.6.19__py3-none-any.whl → 1.7.0a0__py3-none-any.whl

codex/db.py

@@ -0,0 +1,119 @@
+"""Low level database utilities."""
+
+import re
+import sqlite3
+
+from django.core.management import call_command
+from django.db import DEFAULT_DB_ALIAS, connection, connections
+from django.db.migrations.executor import MigrationExecutor
+
+from codex.librarian.janitor.integrity import (
+    cleanup_custom_cover_libraries,
+    fix_foreign_keys,
+    fts_integrity_check,
+    fts_rebuild,
+    integrity_check,
+)
+from codex.librarian.janitor.janitor import Janitor
+from codex.librarian.mp_queue import LIBRARIAN_QUEUE
+from codex.logger.logging import get_logger
+from codex.logger.mp_queue import LOG_QUEUE
+from codex.settings.settings import (
+    BACKUP_DB_PATH,
+    CONFIG_PATH,
+    DB_PATH,
+    FIX_FOREIGN_KEYS,
+    FTS_INTEGRITY_CHECK,
+    FTS_REBUILD,
+    INTEGRITY_CHECK,
+)
+from codex.version import VERSION
+
+REPAIR_FLAG_PATH = CONFIG_PATH / "rebuild_db"
+DUMP_LINE_MATCHER = re.compile("TRANSACTION|ROLLBACK|COMMIT")
+REBUILT_DB_PATH = DB_PATH.parent / (DB_PATH.name + ".rebuilt")
+
+LOG = get_logger(__name__)
+
+
+def _has_unapplied_migrations():
+    """Check if any migrations are outstanding."""
+    try:
+        connection = connections[DEFAULT_DB_ALIAS]
+        connection.prepare_database()
+        executor = MigrationExecutor(connection)
+        targets = [
+            key for key in executor.loader.graph.leaf_nodes() if key[0] == "codex"
+        ]
+        plan = executor.migration_plan(targets)
+    except Exception as exc:
+        LOG.warning(f"has_unapplied_migrations(): {exc}")
+        return False
+    else:
+        return bool(plan)
+
+
+def _get_backup_db_path(prefix):
+    suffix = f".{prefix}{BACKUP_DB_PATH.suffix}"
+    return BACKUP_DB_PATH.with_suffix(suffix)
+
+
+def _backup_db_before_migration():
+    """If there are migrations to do, backup the db."""
+    backup_path = _get_backup_db_path(f"before-v{VERSION}")
+    janitor = Janitor(LOG_QUEUE, LIBRARIAN_QUEUE)
+    janitor.backup_db(backup_path, show_status=False)
+    LOG.info("Backed up database before migrations")
+
+
+def _repair_db():
+    """Run integrity checks on startup."""
+    if FIX_FOREIGN_KEYS:
+        fix_foreign_keys()
+    if INTEGRITY_CHECK:
+        long = INTEGRITY_CHECK == "full"
+        integrity_check(long)
+    success = fts_integrity_check() if FTS_INTEGRITY_CHECK else True
+    if FTS_REBUILD or not success:
+        fts_rebuild()
+    cleanup_custom_cover_libraries()
+
+
+def _rebuild_db():
+    """Dump and rebuild the database."""
+    # Drastic
+    if not REPAIR_FLAG_PATH.exists():
+        return
+
+    LOG.warning("REBUILDING DATABASE!!")
+    REPAIR_FLAG_PATH.unlink(missing_ok=True)
+    with sqlite3.connect(REBUILT_DB_PATH) as new_db_conn:
+        new_db_cur = new_db_conn.cursor()
+        with sqlite3.connect(DB_PATH) as old_db_conn:
+            for line in old_db_conn.iterdump():
+                if DUMP_LINE_MATCHER.search(line):
+                    continue
+                new_db_cur.execute(line)
+
+    backup_path = _get_backup_db_path("before-rebuild")
+    DB_PATH.rename(backup_path)
+    LOG.info("Backed up old db to %s", backup_path)
+    REBUILT_DB_PATH.replace(DB_PATH)
+    LOG.info("Rebuilt database.")
+
+
+def ensure_db_schema():
+    """Ensure the db is good and up to date."""
+    LOG.info("Ensuring database is correct and up to date...")
+    table_names = connection.introspection.table_names()
+    if db_exists := "django_migrations" in table_names:
+        _rebuild_db()
+        _repair_db()
+
+    if not db_exists or _has_unapplied_migrations():
+        if db_exists:
+            _backup_db_before_migration()
+        call_command("migrate")
+    else:
+        LOG.info("Database up to date.")
+    LOG.info("Database ready.")

codex/librarian/covers/purge.py

@@ -45,7 +45,7 @@ class CoverPurgeMixin(CoverPathMixin):
                 except FileNotFoundError:
                     status.decrement_total()
                 cover_dirs.add(cover_path.parent)
-                self.status_controller.update(status)
+                self.status_controller.update(status, notify=False)
             for cover_dir in cover_dirs:
                 self._cleanup_cover_dirs(cover_dir, cover_root)
             self.log.info(f"Removed {status.complete} cover thumbnails.")

codex/librarian/importer/const.py

@@ -115,7 +115,13 @@ DICT_MODEL_REL_LINK_MAP = MappingProxyType(
 # BULK UPDATE #
 ###############
 
-_EXCLUDEBULK_UPDATE_COMIC_FIELDS = {"bookmark", "created_at", "id", "library"}
+_EXCLUDEBULK_UPDATE_COMIC_FIELDS = {
+    "bookmark",
+    "created_at",
+    "id",
+    "library",
+    "comicfts",
+}
 GROUP_BASE_FIELDS = ("name", "sort_name")
 BULK_UPDATE_COMIC_FIELDS = tuple(
     sorted(

codex/librarian/importer/create_fks.py

@@ -213,8 +213,11 @@ class CreateForeignKeysImporter(CreateCoversImporter):
         # create each depth level first to ensure we can assign parents
         for depth_level in sorted(folder_path_dict):
             paths = sorted(folder_path_dict[depth_level])
-            count += self._bulk_folders_create_depth_level(paths, status)
-            self.log.debug(f"Created {count} Folders at depth level {depth_level}")
+            level_count = self._bulk_folders_create_depth_level(paths, status)
+            count += level_count
+            self.log.debug(
+                f"Created {level_count} Folders at depth level {depth_level}"
+            )
 
         if count:
             self.log.info(f"Created {count} Folders.")

codex/librarian/importer/importerd.py

@@ -12,6 +12,7 @@ from codex.librarian.importer.tasks import (
     LazyImportComicsTask,
     UpdateGroupsTask,
 )
+from codex.librarian.janitor.tasks import JanitorAdoptOrphanFoldersFinishedTask
 from codex.librarian.notifier.tasks import LIBRARY_CHANGED_TASK
 from codex.models import Comic, Folder, Library
 from codex.status import Status
@@ -88,20 +89,24 @@ class ComicImporterThread(QueuedThread):
         importer.bulk_folders_moved()
         return True
 
-    def _adopt_orphan_folders(self):
+    def _adopt_orphan_folders(self, janitor=False):
         """Find orphan folders and move them into their correct place."""
         status = Status(ImportStatusTypes.ADOPT_FOLDERS)
         moved_status = Status(ImportStatusTypes.DIRS_MOVED)
-        self.status_controller.start(status)
-        self.status_controller.start(moved_status)
-        libraries = Library.objects.filter(covers_only=False).only("path")
-        for library in libraries.iterator():
-            folders_left = True
-            while folders_left:
-                # Run until there are no orphan folders
-                folders_left = self._adopt_orphan_folders_for_library(library)
-
-        self.status_controller.finish_many((moved_status, status))
+        try:
+            self.status_controller.start(status)
+            self.status_controller.start(moved_status)
+            libraries = Library.objects.filter(covers_only=False).only("path")
+            for library in libraries.iterator():
+                folders_left = True
+                while folders_left:
+                    # Run until there are no orphan folders
+                    folders_left = self._adopt_orphan_folders_for_library(library)
+        finally:
+            self.status_controller.finish_many((moved_status, status))
+            if janitor:
+                next_task = JanitorAdoptOrphanFoldersFinishedTask()
+                self.librarian_queue.put(next_task)
 
     def process_item(self, item):
         """Run the updater."""
@@ -113,6 +118,6 @@ class ComicImporterThread(QueuedThread):
         elif isinstance(task, UpdateGroupsTask):
             self._update_groups(task)
         elif isinstance(task, AdoptOrphanFoldersTask):
-            self._adopt_orphan_folders()
+            self._adopt_orphan_folders(task.janitor)
         else:
             self.log.warning(f"Bad task sent to library updater {task}")

codex/librarian/importer/moved.py

@@ -164,17 +164,23 @@ class MovedImporter(AggregateMetadataImporter):
         self.status_controller.finish(status)
 
     def _bulk_move_folders(
-        self, src_folder_paths, dest_parent_folders_map, dirs_moved: bidict, status
+        self,
+        src_folder_paths_with_existing_dest_parents,
+        dest_parent_folders_map,
+        dirs_moved: bidict[str, str],
+        status,
     ):
         """Bulk move folders."""
+        # Move collisions removed before this
         folders_to_move = (
-            Folder.objects.filter(library=self.library, path__in=src_folder_paths)
+            Folder.objects.filter(
+                library=self.library,
+                path__in=src_folder_paths_with_existing_dest_parents,
+            )
             .only(*MOVED_BULK_FOLDER_UPDATE_FIELDS)
             .order_by("path")
         )
 
-        # TODO what if a folder with that path already exists in the db?
-
         new_paths = set()
         update_folders = []
         for folder in folders_to_move.iterator():
@@ -201,7 +207,7 @@ class MovedImporter(AggregateMetadataImporter):
         self.status_controller.update(status, notify=False)
 
     def _bulk_move_folders_under_existing_parents(
-        self, dest_parent_folder_paths_map, dirs_moved, status
+        self, dest_parent_folder_paths_map, dirs_moved: bidict, status
     ):
         """Move folders under existing folders."""
         while True:
@@ -214,25 +220,32 @@ class MovedImporter(AggregateMetadataImporter):
 
             # Create a list of folders than can be moved under existing folders.
             dest_parent_folders_map = {}
-            src_folder_paths = []
+            src_folder_paths_with_existing_dest_parents = []
             for extant_parent_folder in extant_parent_folders:
                 dest_parent_folders_map[extant_parent_folder.path] = (
                     extant_parent_folder
                 )
                 if dest_folder_paths := dest_parent_folder_paths_map.pop(
-                    extant_parent_folder.path
+                    extant_parent_folder.path, None
                 ):
                     for dest_folder_path in dest_folder_paths:
                         src_path = dirs_moved.inverse[dest_folder_path]
-                        src_folder_paths.append(src_path)
+                        src_folder_paths_with_existing_dest_parents.append(src_path)
 
-            if not src_folder_paths:
+            if not src_folder_paths_with_existing_dest_parents:
                 return
-            src_folder_paths = sorted(src_folder_paths)
-            self.log.debug(f"Moving folders under existing parents: {src_folder_paths}")
+            src_folder_paths_with_existing_dest_parents = sorted(
+                src_folder_paths_with_existing_dest_parents
+            )
+            self.log.debug(
+                f"Moving folders under existing parents: {src_folder_paths_with_existing_dest_parents}"
+            )
 
             self._bulk_move_folders(
-                src_folder_paths, dest_parent_folders_map, dirs_moved, status
+                src_folder_paths_with_existing_dest_parents,
+                dest_parent_folders_map,
+                dirs_moved,
+                status,
             )
 
     def _get_move_create_folders_one_layer(self, dest_parent_folder_paths_map):
@@ -260,12 +273,30 @@ class MovedImporter(AggregateMetadataImporter):
 
         return frozenset(create_folder_paths_one_layer)
 
+    def _remove_move_collisions(self, dirs_moved: bidict[str, str]):
+        """Remove moves that would collide with an existing Folder."""
+        dest_paths = set(dirs_moved.values())
+        collision_dest_paths = Folder.objects.filter(
+            library=self.library, path__in=dest_paths
+        ).values_list("path", flat=True)
+        if not collision_dest_paths:
+            return
+        collision_dest_paths = sorted(set(collision_dest_paths))
+        for collision_dest_path in collision_dest_paths:
+            dirs_moved.inverse.pop(collision_dest_path, None)
+        self.log.warning(
+            f"Not moving folders to destinations that would collide with existing database folders: {collision_dest_paths}"
+        )
+
     def _bulk_move_folders_and_create_parents(self, status):
         """Find folders that can be moved without creating parents."""
         dirs_moved = bidict(self.task.dirs_moved)
 
+        self._remove_move_collisions(dirs_moved)
+
+        dest_paths = sorted(set(dirs_moved.values()))
         dest_parent_folder_paths_map = {}
-        for dest_path in sorted(set(dirs_moved.values())):
+        for dest_path in dest_paths:
             parent = str(Path(dest_path).parent)
             if parent not in dest_parent_folder_paths_map:
                 dest_parent_folder_paths_map[parent] = set()

codex/librarian/importer/tasks.py

@@ -45,6 +45,8 @@ class LazyImportComicsTask(ImportTask):
 class AdoptOrphanFoldersTask(ImportTask):
     """Move orphaned folders into a correct tree position."""
 
+    janitor: bool = False
+
 
 @dataclass
 class UpdateGroupsTask(ImportTask):

codex/librarian/janitor/cleanup.py

@@ -35,6 +35,7 @@ from codex.models import (
     Team,
     Volume,
 )
+from codex.models.bookmark import Bookmark
 from codex.models.paths import CustomCover
 from codex.status import Status
 from codex.worker_base import WorkerBaseMixin
@@ -84,15 +85,15 @@ CLEANUP_MAP = MappingProxyType(
 class CleanupMixin(WorkerBaseMixin):
     """Cleanup methods for Janitor."""
 
-    def _bulk_cleanup_fks(self, classes, field_name, status):
+    def _bulk_cleanup_fks(self, models, field_name, status):
         """Remove foreign keys that aren't used anymore."""
-        for cls in classes:
+        for model in models:
             filter_dict = {f"{field_name}__isnull": True}
-            query = cls.objects.filter(**filter_dict)
-            count = query.count()
-            query.delete()
+            qs = model.objects.filter(**filter_dict)
+            count = qs.count()
+            qs.delete()
             if count:
-                self.log.info(f"Deleted {count} orphan {cls.__name__}s")
+                self.log.info(f"Deleted {count} orphan {model.__name__}s")
             status.complete += count
             self.status_controller.update(status)
 
@@ -102,8 +103,8 @@ class CleanupMixin(WorkerBaseMixin):
         try:
             self.status_controller.start(status)
             self.log.debug("Cleaning up unused foreign keys...")
-            for field_name, classes in CLEANUP_MAP.items():
-                self._bulk_cleanup_fks(classes, field_name, status)
+            for field_name, models in CLEANUP_MAP.items():
+                self._bulk_cleanup_fks(models, field_name, status)
             level = logging.INFO if status.complete else logging.DEBUG
             self.log.log(level, f"Cleaned up {status.complete} unused foreign keys.")
         finally:
@@ -152,3 +153,15 @@ class CleanupMixin(WorkerBaseMixin):
         finally:
             until = time() + 1
             self.status_controller.finish(status, until=until)
+
+    def cleanup_orphan_bookmarks(self):
+        """Delete bookmarks without users or sessions."""
+        status = Status(JanitorStatusTypes.CLEANUP_BOOKMARKS)
+        try:
+            self.status_controller.start(status)
+            orphan_bms = Bookmark.objects.filter(session=None, user=None)
+            if count := orphan_bms.count():
+                orphan_bms.delete()
+                self.log.info(f"Deleted {count} orphan bookmarks.")
+        finally:
+            self.status_controller.finish(status)