codex 1.7.15__py3-none-any.whl → 1.8.0__py3-none-any.whl

codex/librarian/scribe/janitor/janitor.py

@@ -0,0 +1,133 @@
+"""Janitor task runner."""
+
+from codex.librarian.bookmark.tasks import CodexLatestVersionTask
+from codex.librarian.covers.status import FindOrphanCoversStatus, RemoveCoversStatus
+from codex.librarian.covers.tasks import CoverRemoveOrphansTask
+from codex.librarian.scribe.importer.statii.moved import ImporterMoveFoldersStatus
+from codex.librarian.scribe.janitor.status import (
+    JanitorAdoptOrphanFoldersStatus,
+    JanitorCleanupBookmarksStatus,
+    JanitorCleanupCoversStatus,
+    JanitorCleanupSessionsStatus,
+    JanitorCleanupTagsStatus,
+    JanitorCodexLatestVersionStatus,
+    JanitorDBBackupStatus,
+    JanitorDBFKIntegrityStatus,
+    JanitorDBFTSIntegrityStatus,
+    JanitorDBIntegrityStatus,
+    JanitorDBOptimizeStatus,
+)
+from codex.librarian.scribe.janitor.tasks import (
+    JanitorAdoptOrphanFoldersTask,
+    JanitorBackupTask,
+    JanitorCleanCoversTask,
+    JanitorCleanFKsTask,
+    JanitorCleanupBookmarksTask,
+    JanitorCleanupSessionsTask,
+    JanitorCodexUpdateTask,
+    JanitorForeignKeyCheckTask,
+    JanitorFTSIntegrityCheckTask,
+    JanitorFTSRebuildTask,
+    JanitorImportForceAllFailedTask,
+    JanitorIntegrityCheckTask,
+    JanitorNightlyTask,
+    JanitorVacuumTask,
+)
+from codex.librarian.scribe.janitor.update import JanitorCodexUpdate
+from codex.librarian.scribe.search.status import (
+    SearchIndexCleanStatus,
+    SearchIndexOptimizeStatus,
+    SearchIndexSyncCreateStatus,
+    SearchIndexSyncUpdateStatus,
+)
+from codex.librarian.scribe.search.tasks import (
+    SearchIndexOptimizeTask,
+    SearchIndexSyncTask,
+)
+from codex.models import Timestamp
+
+_JANITOR_STATII = (
+    JanitorCodexLatestVersionStatus,
+    JanitorAdoptOrphanFoldersStatus,
+    ImporterMoveFoldersStatus,
+    JanitorDBFKIntegrityStatus,
+    JanitorDBIntegrityStatus,
+    JanitorDBFTSIntegrityStatus,
+    JanitorCleanupTagsStatus,
+    JanitorCleanupCoversStatus,
+    JanitorCleanupSessionsStatus,
+    JanitorCleanupBookmarksStatus,
+    SearchIndexCleanStatus,
+    SearchIndexSyncUpdateStatus,
+    SearchIndexSyncCreateStatus,
+    SearchIndexOptimizeStatus,
+    JanitorDBOptimizeStatus,
+    JanitorDBBackupStatus,
+    FindOrphanCoversStatus,
+    RemoveCoversStatus,
+)
+
+
+class Janitor(JanitorCodexUpdate):
+    """Janitor inline task runner."""
+
+    def queue_nightly_tasks(self):
+        """Queue all the janitor tasks."""
+        try:
+            self.status_controller.start_many(_JANITOR_STATII)
+            tasks = (
+                CodexLatestVersionTask(),
+                JanitorAdoptOrphanFoldersTask(),
+                JanitorForeignKeyCheckTask(),
+                JanitorIntegrityCheckTask(),
+                JanitorFTSIntegrityCheckTask(),
+                JanitorCleanFKsTask(),
+                JanitorCleanCoversTask(),
+                JanitorCleanupSessionsTask(),
+                JanitorCleanupBookmarksTask(),
+                SearchIndexSyncTask(),
+                SearchIndexOptimizeTask(),
+                JanitorVacuumTask(),
+                JanitorBackupTask(),
+                CoverRemoveOrphansTask(),
+            )
+            for task in tasks:
+                self.librarian_queue.put(task)
+            Timestamp.touch(Timestamp.Choices.JANITOR)
+        except Exception:
+            self.log.exception(f"In {self.__class__.__name__}")
+
+    def handle_task(self, task):  # noqa: PLR0912,C901
+        """Run Janitor tasks as the librarian process directly."""
+        try:
+            match task:
+                case JanitorVacuumTask():
+                    self.vacuum_db()
+                case JanitorBackupTask():
+                    self.backup_db(show_status=True)
+                case JanitorCleanFKsTask():
+                    self.cleanup_fks()
+                case JanitorCleanCoversTask():
+                    self.cleanup_custom_covers()
+                case JanitorCleanupSessionsTask():
+                    self.cleanup_sessions()
+                case JanitorCleanupBookmarksTask():
+                    self.cleanup_orphan_bookmarks()
+                case JanitorImportForceAllFailedTask():
+                    self.force_update_all_failed_imports()
+                case JanitorForeignKeyCheckTask():
+                    self.foreign_key_check()
+                case JanitorIntegrityCheckTask():
+                    self.integrity_check(long=task.long)
+                case JanitorFTSIntegrityCheckTask():
+                    self.fts_integrity_check()
+                case JanitorFTSRebuildTask():
+                    self.fts_rebuild()
+                case JanitorNightlyTask():
+                    self.queue_nightly_tasks()
+                case JanitorCodexUpdateTask():
+                    self.update_codex(force=task.force)
+                case _:
+                    self.log.warning(f"Janitor received unknown task {task}")
+        except Exception:
+            self.log.exception("Janitor task crashed.")

codex/librarian/scribe/janitor/scheduled_time.py

@@ -0,0 +1,13 @@
+"""Janitor Scheduled time."""
+
+from datetime import datetime, time, timedelta
+
+from django.utils import timezone as django_timezone
+from loguru._logger import Logger
+
+
+def get_janitor_time(_log: Logger):
+    """Get midnight relative to now."""
+    tomorrow = django_timezone.now() + timedelta(days=1)
+    tomorrow = tomorrow.astimezone()
+    return datetime.combine(tomorrow, time.min).astimezone()

codex/librarian/scribe/janitor/status.py

@@ -0,0 +1,152 @@
+"""Jantior Statii."""
+
+from abc import ABC
+
+from codex.librarian.scribe.status import ScribeStatus
+
+
+class JanitorStatus(ScribeStatus, ABC):
+    """Jantior Statii."""
+
+
+class JanitorAdoptOrphanFoldersStatus(JanitorStatus):
+    """Janitor Adopt Orphan Folders Status."""
+
+    CODE = "JAF"
+    VERB = "Adopt"
+    _verbed = "Adopted"
+    ITEM_NAME = "orphan folders"
+
+
+class JanitorCleanupTagsStatus(JanitorStatus):
+    """Janitor Cleanup Tags Status."""
+
+    CODE = "JCT"
+    VERB = "Cleanup"
+    _verbed = "Cleaned up"
+    ITEM_NAME = "orphan tags"
+
+
+class JanitorCodexLatestVersionStatus(JanitorStatus):
+    """Janitor Codex Latest Version Status."""
+
+    CODE = "JLV"
+    VERB = "Check"
+    _verbed = "Checked"
+    ITEM_NAME = "Codex latest version"
+    SINGLE = True
+
+
+class JanitorCodexUpdateStatus(JanitorStatus):
+    """Janitor Update Codex Software."""
+
+    CODE = "JCU"
+    VERB = "Update"
+    ITEM_NAME = "Codex server software"
+    SINGLE = True
+    log_success = True
+
+
+class JanitorDBOptimizeStatus(JanitorStatus):
+    """Janitor DB Optimize."""
+
+    CODE = "JDO"
+    VERB = "Optimize"
+    ITEM_NAME = "database"
+    SINGLE = True
+    log_success = True
+
+
+class JanitorDBBackupStatus(JanitorStatus):
+    """Janitor DB Backup."""
+
+    CODE = "JDB"
+    VERB = "Backup"
+    _verbed = "Backed up"
+    ITEM_NAME = "database"
+    SINGLE = True
+
+
+class JanitorCleanupSessionsStatus(JanitorStatus):
+    """Janitor Cleanup Sessions Status."""
+
+    CODE = "JRS"
+    VERB = "Cleanup"
+    _verbed = "Cleaned up"
+    ITEM_NAME = "old sessions"
+
+
+class JanitorCleanupCoversStatus(JanitorStatus):
+    """Janitor Cleanup Covers Status."""
+
+    CODE = "JRV"
+    VERB = "Cleanup"
+    _verbed = "Cleaned up"
+    ITEM_NAME = "orphan covers"
+
+
+class JanitorCleanupBookmarksStatus(JanitorStatus):
+    """Janitor Cleanup Bookmarks Status."""
+
+    CODE = "JRB"
+    VERB = "Cleanup"
+    _verbed = "Cleaned up"
+    ITEM_NAME = "orphan bookmarks"
+
+
+class JanitorDBFKIntegrityStatus(JanitorStatus):
+    """Janitor Check DB FK Integrity Status."""
+
+    CODE = "JIF"
+    VERB = "Check"
+    _verbed = "Checked"
+    ITEM_NAME = "integrtity of database foreign keys"
+    SINGLE = True
+
+
+class JanitorDBIntegrityStatus(JanitorStatus):
+    """Janitor Check DB Integrity Status."""
+
+    CODE = "JID"
+    VERB = "Check"
+    _verbed = "Checked"
+    ITEM_NAME = "integrity of entire database"
+    SINGLE = True
+
+
+class JanitorDBFTSIntegrityStatus(JanitorStatus):
+    """Janitor Check DB FTS Integrity Status."""
+
+    CODE = "JIS"
+    VERB = "Check"
+    _verbed = "Checked"
+    ITEM_NAME = "integrity of full text virtual table"
+    SINGLE = True
+
+
+class JanitorDBFTSRebuildStatus(JanitorStatus):
+    """Janitor Rebuild DB FTS Status."""
+
+    CODE = "JSR"
+    VERB = "Rebuild"
+    _verbed = "Rebuilt"
+    ITEM_NAME = "full text search virtual table"
+    SINGLE = True
+    log_success = True
+
+
+JANITOR_STATII = (
+    JanitorAdoptOrphanFoldersStatus,
+    JanitorCleanupTagsStatus,
+    JanitorCodexUpdateStatus,
+    JanitorCodexLatestVersionStatus,
+    JanitorDBOptimizeStatus,
+    JanitorDBBackupStatus,
+    JanitorCleanupSessionsStatus,
+    JanitorCleanupCoversStatus,
+    JanitorCleanupBookmarksStatus,
+    JanitorDBFKIntegrityStatus,
+    JanitorDBIntegrityStatus,
+    JanitorDBFTSIntegrityStatus,
+    JanitorDBFTSRebuildStatus,
+)

codex/librarian/scribe/janitor/tasks.py

@@ -0,0 +1,71 @@
+"""Janitor Tasks."""
+
+from dataclasses import dataclass
+
+from codex.librarian.scribe.tasks import ScribeTask
+
+
+class JanitorTask(ScribeTask):
+    """Tasks for the janitor."""
+
+
+@dataclass
+class JanitorCodexUpdateTask(JanitorTask):
+    """Task for updater."""
+
+    force: bool = False
+
+
+class JanitorAdoptOrphanFoldersTask(JanitorTask):
+    """Move orphaned folders into a correct tree position."""
+
+
+class JanitorBackupTask(JanitorTask):
+    """Backup the database."""
+
+
+class JanitorVacuumTask(JanitorTask):
+    """Vacuum the database."""
+
+
+class JanitorCleanFKsTask(JanitorTask):
+    """Clean unused foreign keys."""
+
+
+class JanitorCleanCoversTask(JanitorTask):
+    """Clean unused custom covers."""
+
+
+class JanitorCleanupSessionsTask(JanitorTask):
+    """Cleanup Session table."""
+
+
+class JanitorCleanupBookmarksTask(JanitorTask):
+    """Clean unused bookmarks."""
+
+
+class JanitorForeignKeyCheckTask(JanitorTask):
+    """Check and repair foreign keys integrity."""
+
+
+class JanitorImportForceAllFailedTask(JanitorTask):
+    """Force update for failed imports in every library."""
+
+
+@dataclass
+class JanitorIntegrityCheckTask(JanitorTask):
+    """Check integrity and warn."""
+
+    long: bool = True
+
+
+class JanitorFTSIntegrityCheckTask(JanitorTask):
+    """Check fts integrity."""
+
+
+class JanitorFTSRebuildTask(JanitorTask):
+    """Rebuild fts table in place."""
+
+
+class JanitorNightlyTask(JanitorTask):
+    """Submit all janitor nightly tasks to the queue."""

codex/librarian/scribe/janitor/update.py

@@ -0,0 +1,88 @@
+"""Codex auto update."""
+
+import subprocess
+import sys
+
+from versio.version import Version
+
+from codex.choices.admin import AdminFlagChoices
+from codex.librarian.restarter.tasks import CodexRestartTask
+from codex.librarian.scribe.janitor.cleanup import JanitorCleanup
+from codex.librarian.scribe.janitor.status import JanitorCodexUpdateStatus
+from codex.models import AdminFlag
+from codex.models.admin import Timestamp
+from codex.version import VERSION, get_version
+
+
+class JanitorCodexUpdate(JanitorCleanup):
+    """Auto Update codex methods for janitor."""
+
+    def _is_outdated(self):
+        """Is codex outdated."""
+        result = False
+        if not VERSION:
+            self.log.warning("Cannot determine installed Codex version.")
+            return result
+        ts = Timestamp.objects.get(key=Timestamp.Choices.CODEX_VERSION.value)
+        latest_version = ts.version
+        versio_latest_version = Version(latest_version)
+
+        installed_versio_version = Version(VERSION)
+        if versio_latest_version.parts[1] and not installed_versio_version.parts[1]:  # pyright: ignore[reportIndexIssue]
+            pre_blurb = "latest version is a prerelease. But installed version is not."
+        else:
+            result = versio_latest_version > installed_versio_version
+            pre_blurb = ""
+        self.log.debug(f"{latest_version=} > {VERSION=} = {result}{pre_blurb}")
+        return result
+
+    def _update_codex(self, *, force: bool):
+        if force:
+            self.log.info("Forcing update of Codex.")
+        else:
+            eau = AdminFlag.objects.only("on").get(
+                key=AdminFlagChoices.AUTO_UPDATE.value
+            )
+            if not eau.on or not self._is_outdated():
+                self.log.info("Codex is up to date.")
+                return
+
+            self.log.info("Codex seems outdated. Trying to update.")
+
+        args = (
+            sys.executable,
+            "-m",
+            "pip",
+            "install",
+            "--upgrade",
+            "codex",
+        )
+        subprocess.run(  # noqa: S603
+            args,
+            check=True,
+        )
+
+    def update_codex(self, *, force: bool):
+        """Update the package and restart everything if the version changed."""
+        status = JanitorCodexUpdateStatus()
+        try:
+            self.status_controller.start(status)
+            self._update_codex(force=force)
+        except Exception:
+            self.log.exception("Updating Codex software")
+        finally:
+            self.status_controller.finish(status)
+
+        # Restart if changed version.
+        new_version = get_version()
+        restart = new_version != VERSION
+
+        if restart:
+            self.log.success(f"Codex was updated from {VERSION} to {new_version}.")
+            self.librarian_queue.put(CodexRestartTask())
+        else:
+            reason = (
+                "Codex updated to the same version that was previously"
+                f" installed: {VERSION}."
+            )
+            self.log.warning(reason)

codex/librarian/scribe/janitor/vacuum.py

@@ -0,0 +1,53 @@
+"""Vacuum the database."""
+
+from django.db import connection
+from humanize import naturalsize
+
+from codex.librarian.scribe.janitor.integrity import JanitorIntegrity
+from codex.librarian.scribe.janitor.status import (
+    JanitorDBBackupStatus,
+    JanitorDBOptimizeStatus,
+)
+from codex.settings import BACKUP_DB_DIR, BACKUP_DB_PATH, DB_PATH
+
+_OLD_BACKUP_PATH = BACKUP_DB_PATH.with_suffix(BACKUP_DB_PATH.suffix + ".old")
+
+
+class JanitorVacuum(JanitorIntegrity):
+    """Vacuum methods for janitor."""
+
+    def vacuum_db(self):
+        """Vacuum the database and report on savings."""
+        status = JanitorDBOptimizeStatus()
+        try:
+            self.status_controller.start(status)
+            old_size = DB_PATH.stat().st_size
+            with connection.cursor() as cursor:
+                cursor.execute("PRAGMA optimize")
+                cursor.execute("VACUUM")
+                cursor.execute("PRAGMA wal_checkpoint(TRUNCATE)")
+            new_size = DB_PATH.stat().st_size
+            saved = naturalsize(old_size - new_size)
+            self.log.info(f"Vacuumed database. Saved {saved}.")
+        finally:
+            self.status_controller.finish(status)
+
+    def backup_db(self, backup_path=BACKUP_DB_PATH, *, show_status: bool):
+        """Backup the database."""
+        status = JanitorDBBackupStatus() if show_status else None
+        try:
+            if status:
+                self.status_controller.start(status)
+            BACKUP_DB_DIR.mkdir(exist_ok=True, parents=True)
+            if backup_path.is_file():
+                backup_path.replace(_OLD_BACKUP_PATH)
+            path = str(backup_path)
+            with connection.cursor() as cursor:
+                cursor.execute(f"VACUUM INTO {path!r}")
+            _OLD_BACKUP_PATH.unlink(missing_ok=True)
+            self.log.info(f"Backed up database to {path}")
+        except Exception:
+            self.log.exception("Backing up database.")
+        finally:
+            if status:
+                self.status_controller.finish(status)

codex/librarian/scribe/lazy_importer.py

@@ -0,0 +1,51 @@
+"""Kick off an import task for one batch of books."""
+
+from codex.choices.admin import AdminFlagChoices
+from codex.librarian.scribe.importer.tasks import ImportTask
+from codex.librarian.worker import WorkerMixin
+from codex.models.admin import AdminFlag
+from codex.models.comic import Comic
+
+
+class LazyImporter(WorkerMixin):
+    """Kick off an import task for just these books."""
+
+    def lazy_import(self, task):
+        """Kick off an import task for just these books."""
+        if not AdminFlag.objects.get(
+            key=AdminFlagChoices.LAZY_IMPORT_METADATA.value
+        ).on:
+            self.log.debug("Lazy Import disabled by flag.")
+            return
+
+        if task.group == "c":
+            comics = Comic.objects.filter(pk__in=task.pks).only("path", "library_id")
+        elif task.group == "f":
+            comics = Comic.objects.filter(parent_folder__in=task.pks).only(
+                "path", "library_id"
+            )
+        else:
+            self.log.warning(f"No lazy import enabled for group {task}")
+            return
+
+        # Map comics to libraries.
+        library_path_map = {}
+        for comic in comics:
+            library_id = comic.library_id  # pyright: ignore[reportAttributeAccessIssue]
+            if library_id not in library_path_map:
+                library_path_map[library_id] = set()
+            library_path_map[library_id].add(comic.path)
+
+        for library_id, paths in library_path_map.items():
+            # An abridged import task.
+            if files_modified := frozenset(paths):
+                task = ImportTask(
+                    library_id=library_id,
+                    files_modified=files_modified,
+                    force_import_metadata=True,
+                )
+                self.librarian_queue.put(task)
+
+    def __init__(self, *args, **kwargs):
+        """Initialize Worker."""
+        self.init_worker(*args, **kwargs)

codex/librarian/scribe/priority.py

@@ -0,0 +1,69 @@
+"""Priority for Scribe tasks in the PriorityQueue."""
+
+from datetime import datetime, timezone
+
+from codex.librarian.scribe.importer.tasks import (
+    ImportTask,
+)
+from codex.librarian.scribe.janitor.tasks import (
+    JanitorAdoptOrphanFoldersTask,
+    JanitorBackupTask,
+    JanitorCleanCoversTask,
+    JanitorCleanFKsTask,
+    JanitorCleanupBookmarksTask,
+    JanitorCleanupSessionsTask,
+    JanitorCodexUpdateTask,
+    JanitorForeignKeyCheckTask,
+    JanitorFTSIntegrityCheckTask,
+    JanitorFTSRebuildTask,
+    JanitorImportForceAllFailedTask,
+    JanitorIntegrityCheckTask,
+    JanitorNightlyTask,
+    JanitorVacuumTask,
+)
+from codex.librarian.scribe.search.tasks import (
+    SearchIndexCleanStaleTask,
+    SearchIndexClearTask,
+    SearchIndexOptimizeTask,
+    SearchIndexSyncTask,
+)
+from codex.librarian.scribe.tasks import (
+    ImportAbortTask,
+    LazyImportComicsTask,
+    ScribeTask,
+    SearchIndexSyncAbortTask,
+    UpdateGroupsTask,
+)
+
+_SCRIBE_TASK_PRIORITY = (
+    ImportAbortTask,
+    SearchIndexSyncAbortTask,
+    JanitorNightlyTask,
+    JanitorCodexUpdateTask,
+    JanitorAdoptOrphanFoldersTask,
+    JanitorForeignKeyCheckTask,
+    JanitorIntegrityCheckTask,
+    JanitorFTSIntegrityCheckTask,
+    JanitorFTSRebuildTask,
+    JanitorImportForceAllFailedTask,
+    ImportTask,
+    LazyImportComicsTask,
+    UpdateGroupsTask,
+    JanitorCleanFKsTask,
+    JanitorCleanCoversTask,
+    JanitorCleanupSessionsTask,
+    JanitorCleanupBookmarksTask,
+    SearchIndexClearTask,
+    SearchIndexCleanStaleTask,
+    SearchIndexSyncTask,
+    SearchIndexOptimizeTask,
+    JanitorVacuumTask,
+    JanitorBackupTask,
+)
+
+
+def get_task_priority(task: ScribeTask) -> tuple[int, float]:
+    """Get task priority by index."""
+    now = datetime.now(tz=timezone.utc).timestamp()
+    priority = _SCRIBE_TASK_PRIORITY.index(type(task))
+    return priority, now