codex 1.7.15__py3-none-any.whl → 1.8.0__py3-none-any.whl

codex/librarian/scribe/scribed.py

@@ -0,0 +1,130 @@
+"""Bulk import and move comics and folders."""
+
+from multiprocessing import Manager
+from queue import PriorityQueue
+
+from typing_extensions import override
+
+from codex.librarian.scribe.importer.importer import ComicImporter
+from codex.librarian.scribe.importer.tasks import ImportTask
+from codex.librarian.scribe.janitor.adopt_folders import OrphanFolderAdopter
+from codex.librarian.scribe.janitor.janitor import Janitor
+from codex.librarian.scribe.janitor.tasks import (
+    JanitorAdoptOrphanFoldersTask,
+    JanitorFTSRebuildTask,
+    JanitorTask,
+)
+from codex.librarian.scribe.lazy_importer import LazyImporter
+from codex.librarian.scribe.priority import get_task_priority
+from codex.librarian.scribe.search.handler import SearchIndexer
+from codex.librarian.scribe.search.tasks import (
+    SearchIndexClearTask,
+    SearchIndexerTask,
+)
+from codex.librarian.scribe.tasks import (
+    ImportAbortTask,
+    LazyImportComicsTask,
+    SearchIndexSyncAbortTask,
+    UpdateGroupsTask,
+)
+from codex.librarian.scribe.timestamp_update import TimestampUpdater
+from codex.librarian.threads import QueuedThread
+from codex.settings import SEARCH_INDEX_BATCH_SIZE
+
+_ABORT_SEARCH_UPDATE_TASKS = (
+    SearchIndexClearTask,
+    SearchIndexSyncAbortTask,
+    JanitorFTSRebuildTask,
+)
+_ABORT_SEARCH_UPDATE_ON_IMPORT_TASKS = (
+    ImportTask,
+    JanitorAdoptOrphanFoldersTask,
+)
+_ABORT_SEARCH_UPDATE_ON_IMPORT_BATCH_SIZE = 1000
+ABORT_SEARCH_UPDATE_TASKS = (
+    _ABORT_SEARCH_UPDATE_TASKS + _ABORT_SEARCH_UPDATE_ON_IMPORT_TASKS
+    if SEARCH_INDEX_BATCH_SIZE <= _ABORT_SEARCH_UPDATE_ON_IMPORT_BATCH_SIZE
+    else _ABORT_SEARCH_UPDATE_TASKS
+)
+
+
+class ScribeThread(QueuedThread):
+    """A worker to handle all bulk database updates."""
+
+    def __init__(self, *args, **kwargs):
+        """Initialize abort event."""
+        self.abort_import_event = Manager().Event()
+        self.abort_search_update_event = Manager().Event()
+        self.abort_cleanup_event = Manager().Event()
+        super().__init__(*args, queue=PriorityQueue(), **kwargs)
+
+    @override
+    def process_item(self, item):
+        """Run the updater."""
+        task = item[-1]
+        match task:
+            case ImportTask():
+                importer = ComicImporter(
+                    task,
+                    self.log,
+                    self.librarian_queue,
+                    self.db_write_lock,
+                    self.abort_import_event,
+                )
+                importer.apply()
+            case LazyImportComicsTask():
+                worker = LazyImporter(
+                    self.log, self.librarian_queue, self.db_write_lock
+                )
+                worker.lazy_import(task)
+            case UpdateGroupsTask():
+                worker = TimestampUpdater(
+                    self.log, self.librarian_queue, self.db_write_lock
+                )
+                worker.update_groups(task)
+            case JanitorAdoptOrphanFoldersTask():
+                worker = OrphanFolderAdopter(
+                    self.log,
+                    self.librarian_queue,
+                    self.db_write_lock,
+                    event=self.abort_import_event,
+                )
+                worker.adopt_orphan_folders()
+            case SearchIndexerTask():
+                worker = SearchIndexer(
+                    self.log,
+                    self.librarian_queue,
+                    self.db_write_lock,
+                    event=self.abort_search_update_event,
+                )
+                worker.handle_task(task)
+            case JanitorTask():
+                worker = Janitor(
+                    self.log,
+                    self.librarian_queue,
+                    self.db_write_lock,
+                    event=self.abort_cleanup_event,
+                )
+                worker.handle_task(task)
+            case _:
+                self.log.warning(f"Bad task sent to scribe: {task}")
+
+    def put(self, task):
+        """Put item in queue, and signal events."""
+        if isinstance(task, ABORT_SEARCH_UPDATE_TASKS):
+            self.abort_search_update_event.set()
+            if isinstance(task, ImportTask | JanitorAdoptOrphanFoldersTask):
+                self.abort_cleanup_event.set()
+                self.log.debug("Abort cleanup db signal given.")
+            elif isinstance(task, SearchIndexSyncAbortTask):
+                self.log.debug(
+                    "Search Index Update abort signal given. May take a bit for the current import subtask to finish."
+                )
+                return
+        elif isinstance(task, ImportAbortTask):
+            self.abort_import_event.set()
+            self.log.debug("Import abort signal given.")
+            return
+        priority = get_task_priority(task)
+        item = (priority, task)
+        self.queue.put(item)

codex/librarian/scribe/search/__init__.py

@@ -0,0 +1 @@
+"""Search Indexer."""

codex/librarian/scribe/search/handler.py

@@ -0,0 +1,28 @@
+"""Handle search indexer tasks."""
+
+from codex.librarian.scribe.search.sync import SearchIndexerSync
+from codex.librarian.scribe.search.tasks import (
+    SearchIndexCleanStaleTask,
+    SearchIndexClearTask,
+    SearchIndexerTask,
+    SearchIndexOptimizeTask,
+    SearchIndexSyncTask,
+)
+
+
+class SearchIndexer(SearchIndexerSync):
+    """Handle search indexer tasks."""
+
+    def handle_task(self, task: SearchIndexerTask):
+        """Handle search indexer tasks."""
+        match task:
+            case SearchIndexSyncTask():
+                self.update_search_index(rebuild=task.rebuild)
+            case SearchIndexCleanStaleTask():
+                self.remove_stale_records()
+            case SearchIndexOptimizeTask():
+                self.optimize()
+            case SearchIndexClearTask():
+                self.clear_search_index()
+            case _:
+                self.log.warning(f"Bad task sent to scribe {task}")

codex/librarian/scribe/search/optimize.py

@@ -0,0 +1,30 @@
+"""Search Index cleanup."""
+
+from django.db import connection
+
+from codex.librarian.scribe.search.status import SearchIndexOptimizeStatus
+from codex.librarian.worker import WorkerStatusMixin
+
+_TABLE = "codex_comicfts"
+_OPTIMIZE_SQL = f"INSERT INTO {_TABLE}({_TABLE}) VALUES('optimize')"
+
+
+class SearchIndexerOptimize(WorkerStatusMixin):
+    """Search Index optimize methods."""
+
+    def __init__(self, *args, event, **kwargs):
+        """Initialize search engine."""
+        self.abort_event = event
+        self.init_worker(*args, **kwargs)
+
+    def optimize(self):
+        """Remove records not in the database from the index, trapping exceptions."""
+        status = SearchIndexOptimizeStatus()
+        try:
+            self.status_controller.start(status)
+            with connection.cursor() as cursor:
+                cursor.execute(_OPTIMIZE_SQL)
+        except Exception:
+            self.log.exception("Optimizing search index:")
+        finally:
+            self.status_controller.finish(status)

codex/librarian/scribe/search/remove.py

@@ -0,0 +1,44 @@
+"""Search Index cleanup."""
+
+from codex.librarian.scribe.search.optimize import SearchIndexerOptimize
+from codex.librarian.scribe.search.status import (
+    SearchIndexCleanStatus,
+    SearchIndexClearStatus,
+)
+from codex.models.comic import ComicFTS
+
+
+class SearchIndexerRemove(SearchIndexerOptimize):
+    """Search Index cleanup methods."""
+
+    def clear_search_index(self):
+        """Clear the search index."""
+        clear_status = SearchIndexClearStatus()
+        self.status_controller.start(clear_status)
+        ComicFTS.objects.all().delete()
+        self.status_controller.finish(clear_status)
+
+    def _remove_stale_records(self, status):
+        """Remove records not in the database from the index."""
+        self.status_controller.start(status)
+        self.log.debug("Finding stale records to remove...")
+        delete_comicfts = ComicFTS.objects.filter(comic__isnull=True)
+        status.total = delete_comicfts.count()
+        self.status_controller.update(status)
+        if status.total:
+            self.log.debug(f"Removing {status.total} stale records...")
+        count, _ = delete_comicfts.delete()
+        status.complete = count
+        return count
+
+    def remove_stale_records(self, *, log_success: bool = True) -> int:
+        """Remove records not in the database from the index, trapping exceptions."""
+        count = 0
+        status = SearchIndexCleanStatus(log_success=log_success)
+        try:
+            count = self._remove_stale_records(status)
+        except Exception:
+            self.log.exception("Removing stale records:")
+        finally:
+            self.status_controller.finish(status)
+        return count

codex/librarian/scribe/search/status.py

@@ -0,0 +1,66 @@
+"""Search Index Sync Statii."""
+
+from abc import ABC
+
+from codex.librarian.scribe.status import ScribeStatus
+
+
+class SearchIndexStatus(ScribeStatus, ABC):
+    """Search Index Sync Statii."""
+
+
+class SearchIndexClearStatus(SearchIndexStatus):
+    """Search Index Clear Status."""
+
+    CODE = "SIX"
+    VERB = "Clear"
+    _verbed = "Cleared"
+    ITEM_NAME = "full text search table"
+    SINGLE = True
+    log_success = True
+
+
+class SearchIndexCleanStatus(SearchIndexStatus):
+    """Search Index Clean Status."""
+
+    CODE = "SIR"
+    VERB = "Clean"
+    _verbed = "Cleaned"
+    ITEM_NAME = "orphan search entries"
+
+
+class SearchIndexOptimizeStatus(SearchIndexStatus):
+    """Search Index Optimize Status."""
+
+    CODE = "SIO"
+    VERB = "Optimize"
+    ITEM_NAME = "search virtual table"
+    SINGLE = True
+    log_success = True
+
+
+class SearchIndexSyncUpdateStatus(SearchIndexStatus):
+    """Search Index Sync Update Status."""
+
+    CODE = "SSU"
+    VERB = "Sync"
+    _verbed = "Synced"
+    ITEM_NAME = "old search entries"
+
+
+class SearchIndexSyncCreateStatus(SearchIndexStatus):
+    """Search Index Sync Create Status."""
+
+    CODE = "SSC"
+    VERB = "Sync"
+    _verbed = "Synced"
+    ITEM_NAME = "new search entries"
+
+
+SEARCH_INDEX_STATII = (
+    SearchIndexClearStatus,
+    SearchIndexCleanStatus,
+    SearchIndexOptimizeStatus,
+    SearchIndexSyncUpdateStatus,
+    SearchIndexSyncCreateStatus,
+)