codex 1.7.14__py3-none-any.whl → 1.8.0a2__py3-none-any.whl

codex/librarian/scribe/janitor/tasks.py

@@ -0,0 +1,71 @@
+"""Janitor Tasks."""
+
+from dataclasses import dataclass
+
+from codex.librarian.scribe.tasks import ScribeTask
+
+
+class JanitorTask(ScribeTask):
+    """Tasks for the janitor."""
+
+
+@dataclass
+class JanitorCodexUpdateTask(JanitorTask):
+    """Task for updater."""
+
+    force: bool = False
+
+
+class JanitorAdoptOrphanFoldersTask(JanitorTask):
+    """Move orphaned folders into a correct tree position."""
+
+
+class JanitorBackupTask(JanitorTask):
+    """Backup the database."""
+
+
+class JanitorVacuumTask(JanitorTask):
+    """Vacuum the database."""
+
+
+class JanitorCleanFKsTask(JanitorTask):
+    """Clean unused foreign keys."""
+
+
+class JanitorCleanCoversTask(JanitorTask):
+    """Clean unused custom covers."""
+
+
+class JanitorCleanupSessionsTask(JanitorTask):
+    """Cleanup Session table."""
+
+
+class JanitorCleanupBookmarksTask(JanitorTask):
+    """Clean unused bookmarks."""
+
+
+class JanitorForeignKeyCheckTask(JanitorTask):
+    """Check and repair foreign keys integrity."""
+
+
+class JanitorImportForceAllFailedTask(JanitorTask):
+    """Force update for failed imports in every library."""
+
+
+@dataclass
+class JanitorIntegrityCheckTask(JanitorTask):
+    """Check integrity and warn."""
+
+    long: bool = True
+
+
+class JanitorFTSIntegrityCheckTask(JanitorTask):
+    """Check fts integrity."""
+
+
+class JanitorFTSRebuildTask(JanitorTask):
+    """Rebuild fts table in place."""
+
+
+class JanitorNightlyTask(JanitorTask):
+    """Submit all janitor nightly tasks to the queue."""

codex/librarian/scribe/janitor/update.py

@@ -0,0 +1,88 @@
+"""Codex auto update."""
+
+import subprocess
+import sys
+
+from versio.version import Version
+
+from codex.choices.admin import AdminFlagChoices
+from codex.librarian.restarter.tasks import CodexRestartTask
+from codex.librarian.scribe.janitor.cleanup import JanitorCleanup
+from codex.librarian.scribe.janitor.status import JanitorCodexUpdateStatus
+from codex.models import AdminFlag
+from codex.models.admin import Timestamp
+from codex.version import VERSION, get_version
+
+
+class JanitorCodexUpdate(JanitorCleanup):
+    """Auto Update codex methods for janitor."""
+
+    def _is_outdated(self):
+        """Is codex outdated."""
+        result = False
+        if not VERSION:
+            self.log.warning("Cannot determine installed Codex version.")
+            return result
+        ts = Timestamp.objects.get(key=Timestamp.Choices.CODEX_VERSION.value)
+        latest_version = ts.version
+        versio_latest_version = Version(latest_version)
+
+        installed_versio_version = Version(VERSION)
+        if versio_latest_version.parts[1] and not installed_versio_version.parts[1]:  # pyright: ignore[reportIndexIssue]
+            pre_blurb = "latest version is a prerelease. But installed version is not."
+        else:
+            result = versio_latest_version > installed_versio_version
+            pre_blurb = ""
+        self.log.debug(f"{latest_version=} > {VERSION=} = {result}{pre_blurb}")
+        return result
+
+    def _update_codex(self, *, force: bool):
+        if force:
+            self.log.info("Forcing update of Codex.")
+        else:
+            eau = AdminFlag.objects.only("on").get(
+                key=AdminFlagChoices.AUTO_UPDATE.value
+            )
+            if not eau.on or not self._is_outdated():
+                self.log.info("Codex is up to date.")
+                return
+
+            self.log.info("Codex seems outdated. Trying to update.")
+
+        args = (
+            sys.executable,
+            "-m",
+            "pip",
+            "install",
+            "--upgrade",
+            "codex",
+        )
+        subprocess.run(  # noqa: S603
+            args,
+            check=True,
+        )
+
+    def update_codex(self, *, force: bool):
+        """Update the package and restart everything if the version changed."""
+        status = JanitorCodexUpdateStatus()
+        try:
+            self.status_controller.start(status)
+            self._update_codex(force=force)
+        except Exception:
+            self.log.exception("Updating Codex software")
+        finally:
+            self.status_controller.finish(status)
+
+        # Restart if changed version.
+        new_version = get_version()
+        restart = new_version != VERSION
+
+        if restart:
+            self.log.success(f"Codex was updated from {VERSION} to {new_version}.")
+            self.librarian_queue.put(CodexRestartTask())
+        else:
+            reason = (
+                "Codex updated to the same version that was previously"
+                f" installed: {VERSION}."
+            )
+            self.log.warning(reason)

codex/librarian/scribe/janitor/vacuum.py

@@ -0,0 +1,53 @@
+"""Vacuum the database."""
+
+from django.db import connection
+from humanize import naturalsize
+
+from codex.librarian.scribe.janitor.integrity import JanitorIntegrity
+from codex.librarian.scribe.janitor.status import (
+    JanitorDBBackupStatus,
+    JanitorDBOptimizeStatus,
+)
+from codex.settings import BACKUP_DB_DIR, BACKUP_DB_PATH, DB_PATH
+
+_OLD_BACKUP_PATH = BACKUP_DB_PATH.with_suffix(BACKUP_DB_PATH.suffix + ".old")
+
+
+class JanitorVacuum(JanitorIntegrity):
+    """Vacuum methods for janitor."""
+
+    def vacuum_db(self):
+        """Vacuum the database and report on savings."""
+        status = JanitorDBOptimizeStatus()
+        try:
+            self.status_controller.start(status)
+            old_size = DB_PATH.stat().st_size
+            with connection.cursor() as cursor:
+                cursor.execute("PRAGMA optimize")
+                cursor.execute("VACUUM")
+                cursor.execute("PRAGMA wal_checkpoint(TRUNCATE)")
+            new_size = DB_PATH.stat().st_size
+            saved = naturalsize(old_size - new_size)
+            self.log.info(f"Vacuumed database. Saved {saved}.")
+        finally:
+            self.status_controller.finish(status)
+
+    def backup_db(self, backup_path=BACKUP_DB_PATH, *, show_status: bool):
+        """Backup the database."""
+        status = JanitorDBBackupStatus() if show_status else None
+        try:
+            if status:
+                self.status_controller.start(status)
+            BACKUP_DB_DIR.mkdir(exist_ok=True, parents=True)
+            if backup_path.is_file():
+                backup_path.replace(_OLD_BACKUP_PATH)
+            path = str(backup_path)
+            with connection.cursor() as cursor:
+                cursor.execute(f"VACUUM INTO {path!r}")
+            _OLD_BACKUP_PATH.unlink(missing_ok=True)
+            self.log.info(f"Backed up database to {path}")
+        except Exception:
+            self.log.exception("Backing up database.")
+        finally:
+            if status:
+                self.status_controller.finish(status)

codex/librarian/scribe/lazy_importer.py

@@ -0,0 +1,51 @@
+"""Kick off an import task for one batch of books."""
+
+from codex.choices.admin import AdminFlagChoices
+from codex.librarian.scribe.importer.tasks import ImportTask
+from codex.librarian.worker import WorkerMixin
+from codex.models.admin import AdminFlag
+from codex.models.comic import Comic
+
+
+class LazyImporter(WorkerMixin):
+    """Kick off an import task for just these books."""
+
+    def lazy_import(self, task):
+        """Kick off an import task for just these books."""
+        if not AdminFlag.objects.get(
+            key=AdminFlagChoices.LAZY_IMPORT_METADATA.value
+        ).on:
+            self.log.debug("Lazy Import disabled by flag.")
+            return
+
+        if task.group == "c":
+            comics = Comic.objects.filter(pk__in=task.pks).only("path", "library_id")
+        elif task.group == "f":
+            comics = Comic.objects.filter(parent_folder__in=task.pks).only(
+                "path", "library_id"
+            )
+        else:
+            self.log.warning(f"No lazy import enabled for group {task}")
+            return
+
+        # Map comics to libraries.
+        library_path_map = {}
+        for comic in comics:
+            library_id = comic.library_id  # pyright: ignore[reportAttributeAccessIssue]
+            if library_id not in library_path_map:
+                library_path_map[library_id] = set()
+            library_path_map[library_id].add(comic.path)
+
+        for library_id, paths in library_path_map.items():
+            # An abridged import task.
+            if files_modified := frozenset(paths):
+                task = ImportTask(
+                    library_id=library_id,
+                    files_modified=files_modified,
+                    force_import_metadata=True,
+                )
+                self.librarian_queue.put(task)
+
+    def __init__(self, *args, **kwargs):
+        """Initialize Worker."""
+        self.init_worker(*args, **kwargs)

codex/librarian/scribe/priority.py

@@ -0,0 +1,65 @@
+"""Priority for Scribe tasks in the PriorityQueue."""
+
+from codex.librarian.scribe.importer.tasks import (
+    ImportTask,
+)
+from codex.librarian.scribe.janitor.tasks import (
+    JanitorAdoptOrphanFoldersTask,
+    JanitorBackupTask,
+    JanitorCleanCoversTask,
+    JanitorCleanFKsTask,
+    JanitorCleanupBookmarksTask,
+    JanitorCleanupSessionsTask,
+    JanitorCodexUpdateTask,
+    JanitorForeignKeyCheckTask,
+    JanitorFTSIntegrityCheckTask,
+    JanitorFTSRebuildTask,
+    JanitorImportForceAllFailedTask,
+    JanitorIntegrityCheckTask,
+    JanitorNightlyTask,
+    JanitorVacuumTask,
+)
+from codex.librarian.scribe.search.tasks import (
+    SearchIndexCleanStaleTask,
+    SearchIndexClearTask,
+    SearchIndexOptimizeTask,
+    SearchIndexSyncTask,
+)
+from codex.librarian.scribe.tasks import (
+    ImportAbortTask,
+    LazyImportComicsTask,
+    ScribeTask,
+    SearchIndexSyncAbortTask,
+    UpdateGroupsTask,
+)
+
+_SCRIBE_TASK_PRIORITY = (
+    ImportAbortTask,
+    SearchIndexSyncAbortTask,
+    JanitorNightlyTask,
+    JanitorCodexUpdateTask,
+    JanitorAdoptOrphanFoldersTask,
+    JanitorForeignKeyCheckTask,
+    JanitorIntegrityCheckTask,
+    JanitorFTSIntegrityCheckTask,
+    JanitorFTSRebuildTask,
+    JanitorImportForceAllFailedTask,
+    ImportTask,
+    LazyImportComicsTask,
+    UpdateGroupsTask,
+    JanitorCleanFKsTask,
+    JanitorCleanCoversTask,
+    JanitorCleanupSessionsTask,
+    JanitorCleanupBookmarksTask,
+    SearchIndexClearTask,
+    SearchIndexCleanStaleTask,
+    SearchIndexSyncTask,
+    SearchIndexOptimizeTask,
+    JanitorVacuumTask,
+    JanitorBackupTask,
+)
+
+
+def get_task_priority(task: ScribeTask):
+    """Get task priority by index."""
+    return _SCRIBE_TASK_PRIORITY.index(type(task))

codex/librarian/scribe/scribed.py

@@ -0,0 +1,130 @@
+"""Bulk import and move comics and folders."""
+
+from multiprocessing import Manager
+from queue import PriorityQueue
+
+from typing_extensions import override
+
+from codex.librarian.scribe.importer.importer import ComicImporter
+from codex.librarian.scribe.importer.tasks import ImportTask
+from codex.librarian.scribe.janitor.adopt_folders import OrphanFolderAdopter
+from codex.librarian.scribe.janitor.janitor import Janitor
+from codex.librarian.scribe.janitor.tasks import (
+    JanitorAdoptOrphanFoldersTask,
+    JanitorFTSRebuildTask,
+    JanitorTask,
+)
+from codex.librarian.scribe.lazy_importer import LazyImporter
+from codex.librarian.scribe.priority import get_task_priority
+from codex.librarian.scribe.search.handler import SearchIndexer
+from codex.librarian.scribe.search.tasks import (
+    SearchIndexClearTask,
+    SearchIndexerTask,
+)
+from codex.librarian.scribe.tasks import (
+    ImportAbortTask,
+    LazyImportComicsTask,
+    SearchIndexSyncAbortTask,
+    UpdateGroupsTask,
+)
+from codex.librarian.scribe.timestamp_update import TimestampUpdater
+from codex.librarian.threads import QueuedThread
+from codex.settings import SEARCH_INDEX_BATCH_SIZE
+
+_ABORT_SEARCH_UPDATE_TASKS = (
+    SearchIndexClearTask,
+    SearchIndexSyncAbortTask,
+    JanitorFTSRebuildTask,
+)
+_ABORT_SEARCH_UPDATE_ON_IMPORT_TASKS = (
+    ImportTask,
+    JanitorAdoptOrphanFoldersTask,
+)
+_ABORT_SEARCH_UPDATE_ON_IMPORT_BATCH_SIZE = 1000
+ABORT_SEARCH_UPDATE_TASKS = (
+    _ABORT_SEARCH_UPDATE_TASKS + _ABORT_SEARCH_UPDATE_ON_IMPORT_TASKS
+    if SEARCH_INDEX_BATCH_SIZE <= _ABORT_SEARCH_UPDATE_ON_IMPORT_BATCH_SIZE
+    else _ABORT_SEARCH_UPDATE_TASKS
+)
+
+
+class ScribeThread(QueuedThread):
+    """A worker to handle all bulk database updates."""
+
+    def __init__(self, *args, **kwargs):
+        """Initialize abort event."""
+        self.abort_import_event = Manager().Event()
+        self.abort_search_update_event = Manager().Event()
+        self.abort_cleanup_event = Manager().Event()
+        super().__init__(*args, queue=PriorityQueue(), **kwargs)
+
+    @override
+    def process_item(self, item):
+        """Run the updater."""
+        task = item[1]
+        match task:
+            case ImportTask():
+                importer = ComicImporter(
+                    task,
+                    self.log,
+                    self.librarian_queue,
+                    self.db_write_lock,
+                    self.abort_import_event,
+                )
+                importer.apply()
+            case LazyImportComicsTask():
+                worker = LazyImporter(
+                    self.log, self.librarian_queue, self.db_write_lock
+                )
+                worker.lazy_import(task)
+            case UpdateGroupsTask():
+                worker = TimestampUpdater(
+                    self.log, self.librarian_queue, self.db_write_lock
+                )
+                worker.update_groups(task)
+            case JanitorAdoptOrphanFoldersTask():
+                worker = OrphanFolderAdopter(
+                    self.log,
+                    self.librarian_queue,
+                    self.db_write_lock,
+                    event=self.abort_import_event,
+                )
+                worker.adopt_orphan_folders()
+            case SearchIndexerTask():
+                worker = SearchIndexer(
+                    self.log,
+                    self.librarian_queue,
+                    self.db_write_lock,
+                    event=self.abort_search_update_event,
+                )
+                worker.handle_task(task)
+            case JanitorTask():
+                worker = Janitor(
+                    self.log,
+                    self.librarian_queue,
+                    self.db_write_lock,
+                    event=self.abort_cleanup_event,
+                )
+                worker.handle_task(task)
+            case _:
+                self.log.warning(f"Bad task sent to scribe: {task}")
+
+    def put(self, task):
+        """Put item in queue, and signal events."""
+        if isinstance(task, ABORT_SEARCH_UPDATE_TASKS):
+            self.abort_search_update_event.set()
+            if isinstance(task, ImportTask | JanitorAdoptOrphanFoldersTask):
+                self.abort_cleanup_event.set()
+                self.log.debug("Abort cleanup db signal given.")
+            elif isinstance(task, SearchIndexSyncAbortTask):
+                self.log.debug(
+                    "Search Index Update abort signal given. May take a bit for the current import subtask to finish."
+                )
+                return
+        elif isinstance(task, ImportAbortTask):
+            self.abort_import_event.set()
+            self.log.debug("Import abort signal given.")
+            return
+        priority = get_task_priority(task)
+        item = (priority, task)
+        self.queue.put(item)

codex/librarian/scribe/search/__init__.py

@@ -0,0 +1 @@
+"""Search Indexer."""

codex/librarian/scribe/search/handler.py

@@ -0,0 +1,28 @@
+"""Handle search indexer tasks."""
+
+from codex.librarian.scribe.search.sync import SearchIndexerSync
+from codex.librarian.scribe.search.tasks import (
+    SearchIndexCleanStaleTask,
+    SearchIndexClearTask,
+    SearchIndexerTask,
+    SearchIndexOptimizeTask,
+    SearchIndexSyncTask,
+)
+
+
+class SearchIndexer(SearchIndexerSync):
+    """Handle search indexer tasks."""
+
+    def handle_task(self, task: SearchIndexerTask):
+        """Handle search indexer tasks."""
+        match task:
+            case SearchIndexSyncTask():
+                self.update_search_index(rebuild=task.rebuild)
+            case SearchIndexCleanStaleTask():
+                self.remove_stale_records()
+            case SearchIndexOptimizeTask():
+                self.optimize()
+            case SearchIndexClearTask():
+                self.clear_search_index()
+            case _:
+                self.log.warning(f"Bad task sent to scribe {task}")

codex/librarian/scribe/search/optimize.py

@@ -0,0 +1,31 @@
+"""Search Index cleanup."""
+
+from django.db import connection
+
+from codex.librarian.scribe.search.status import SearchIndexOptimizeStatus
+from codex.librarian.worker import WorkerStatusMixin
+
+_TABLE = "codex_comicfts"
+_OPTIMIZE_SQL = f"INSERT INTO {_TABLE}({_TABLE}) VALUES('optimize')"
+
+
+class SearchIndexerOptimize(WorkerStatusMixin):
+    """Search Index optimize methods."""
+
+    def __init__(self, *args, event, **kwargs):
+        """Initialize search engine."""
+        self.abort_event = event
+        self.init_worker(*args, **kwargs)
+
+    def optimize(self):
+        """Remove records not in the database from the index, trapping exceptions."""
+        status = SearchIndexOptimizeStatus()
+        try:
+            self.status_controller.update(status)
+            self.log.debug("Optimizing search index...")
+            with connection.cursor() as cursor:
+                cursor.execute(_OPTIMIZE_SQL)
+        except Exception:
+            self.log.exception("Optimizing search index:")
+        finally:
+            self.status_controller.finish(status)

codex/librarian/scribe/search/remove.py

@@ -0,0 +1,44 @@
+"""Search Index cleanup."""
+
+from codex.librarian.scribe.search.optimize import SearchIndexerOptimize
+from codex.librarian.scribe.search.status import (
+    SearchIndexCleanStatus,
+    SearchIndexClearStatus,
+)
+from codex.models.comic import ComicFTS
+
+
+class SearchIndexerRemove(SearchIndexerOptimize):
+    """Search Index cleanup methods."""
+
+    def clear_search_index(self):
+        """Clear the search index."""
+        clear_status = SearchIndexClearStatus()
+        self.status_controller.start(clear_status)
+        ComicFTS.objects.all().delete()
+        self.status_controller.finish(clear_status)
+
+    def _remove_stale_records(self, status):
+        """Remove records not in the database from the index."""
+        self.status_controller.start(status)
+        self.log.debug("Finding stale records to remove...")
+        delete_comicfts = ComicFTS.objects.filter(comic__isnull=True)
+        status.total = delete_comicfts.count()
+        self.status_controller.update(status)
+        if status.total:
+            self.log.debug(f"Removing {status.total} stale records...")
+        count, _ = delete_comicfts.delete()
+        status.complete = count
+        return count
+
+    def remove_stale_records(self) -> int:
+        """Remove records not in the database from the index, trapping exceptions."""
+        count = 0
+        status = SearchIndexCleanStatus()
+        try:
+            count = self._remove_stale_records(status)
+        except Exception:
+            self.log.exception("Removing stale records:")
+        finally:
+            self.status_controller.finish(status)
+        return count

codex/librarian/scribe/search/status.py

@@ -0,0 +1,66 @@
+"""Search Index Sync Statii."""
+
+from abc import ABC
+
+from codex.librarian.scribe.status import ScribeStatus
+
+
+class SearchIndexStatus(ScribeStatus, ABC):
+    """Search Index Sync Statii."""
+
+
+class SearchIndexClearStatus(SearchIndexStatus):
+    """Search Index Clear Status."""
+
+    CODE = "SIX"
+    VERB = "Clear"
+    _verbed = "Cleared"
+    ITEM_NAME = "full text search table"
+    SINGLE = True
+    LOG_SUCCESS = True
+
+
+class SearchIndexCleanStatus(SearchIndexStatus):
+    """Search Index Clean Status."""
+
+    CODE = "SIR"
+    VERB = "Clean"
+    _verbed = "Cleaned"
+    ITEM_NAME = "orphan search entries"
+
+
+class SearchIndexOptimizeStatus(SearchIndexStatus):
+    """Search Index Optimize Status."""
+
+    CODE = "SIO"
+    VERB = "Optimize"
+    ITEM_NAME = "search virtual table"
+    SINGLE = True
+    LOG_SUCCESS = True
+
+
+class SearchIndexSyncUpdateStatus(SearchIndexStatus):
+    """Search Index Sync Update Status."""
+
+    CODE = "SSU"
+    VERB = "Sync old"
+    _verbed = "Synced old"
+    ITEM_NAME = "search entries"
+
+
+class SearchIndexSyncCreateStatus(SearchIndexStatus):
+    """Search Index Sync Create Status."""
+
+    CODE = "SSC"
+    VERB = "Sync new"
+    _verbed = "Synced new"
+    ITEM_NAME = "search entries"
+
+
+SEARCH_INDEX_STATII = (
+    SearchIndexClearStatus,
+    SearchIndexCleanStatus,
+    SearchIndexOptimizeStatus,
+    SearchIndexSyncUpdateStatus,
+    SearchIndexSyncCreateStatus,
+)