codex 1.6.18__py3-none-any.whl → 1.7.0__py3-none-any.whl

codex/librarian/librariand.py

@@ -7,6 +7,8 @@ from typing import NamedTuple
 
 from caseconverter import snakecase
 
+from codex.librarian.bookmark.bookmarkd import BookmarkThread
+from codex.librarian.bookmark.tasks import BookmarkTask
 from codex.librarian.covers.coverd import CoverThread
 from codex.librarian.covers.tasks import CoverTask
 from codex.librarian.cron.crond import CronThread
@@ -24,7 +26,7 @@ from codex.librarian.search.searchd import SearchIndexerThread
 from codex.librarian.search.tasks import (
     SearchIndexAbortTask,
     SearchIndexerTask,
-    SearchIndexRebuildIfDBChangedTask,
+    SearchIndexUpdateTask,
 )
 from codex.librarian.tasks import DelayedTasks, LibrarianShutdownTask, WakeCronTask
 from codex.librarian.telemeter.tasks import TelemeterTask
@@ -46,6 +48,7 @@ class LibrarianDaemon(Process, LoggerBaseMixin):
     """Librarian Process."""
 
     _THREAD_CLASSES = (
+        BookmarkThread,
         NotifierThread,
         DelayedTasksThread,
         CoverThread,
@@ -76,7 +79,7 @@ class LibrarianDaemon(Process, LoggerBaseMixin):
         startup_tasks = (
             AdoptOrphanFoldersTask(),
             WatchdogSyncTask(),
-            SearchIndexRebuildIfDBChangedTask(),
+            SearchIndexUpdateTask(False),
         )
 
         for task in startup_tasks:
@@ -88,6 +91,8 @@ class LibrarianDaemon(Process, LoggerBaseMixin):
         match task:
             case CoverTask():
                 self._threads.cover_thread.queue.put(task)
+            case BookmarkTask():
+                self._threads.bookmark_thread.queue.put(task)
             case WatchdogEventTask():
                 self._threads.watchdog_event_batcher_thread.queue.put(task)
             case ImportTask():
@@ -101,12 +106,12 @@ class LibrarianDaemon(Process, LoggerBaseMixin):
                 self._threads.library_polling_observer.poll(
                     task.library_ids, task.force
                 )
+            case SearchIndexAbortTask():
+                # Must come before the generic SearchIndexerTask below
+                self.search_indexer_abort_event.set()
+                self.log.debug("Told search indexers to stop for db updates.")
             case SearchIndexerTask():
-                if isinstance(task, SearchIndexAbortTask):
-                    self.search_indexer_abort_event.set()
-                    self.log.debug("Told search indexers to stop for db updates.")
-                else:
-                    self._threads.search_indexer_thread.queue.put(task)
+                self._threads.search_indexer_thread.queue.put(task)
             case WakeCronTask():
                 self._threads.cron_thread.end_timeout()
             case TelemeterTask():
@@ -150,6 +155,16 @@ class LibrarianDaemon(Process, LoggerBaseMixin):
             thread.start()
         self.log.info(f"{self.__class__.__name__} started all threads.")
 
+    def _startup(self):
+        """Initialize threads."""
+        self.init_logger(self.log_queue)
+        self.log.debug(f"Started {self.__class__.__name__}.")
+        self.janitor = Janitor(self.log_queue, self.queue)
+        self._create_threads()  # can't do this in init.
+        self._start_threads()
+        self.run_loop = True
+        self.log.info(f"{self.__class__.__name__} ready for tasks.")
+
     def _stop_threads(self):
         """Stop all librarian's threads."""
         self.log.debug(f"{self.__class__.__name__} stopping all threads...")
@@ -164,19 +179,26 @@ class LibrarianDaemon(Process, LoggerBaseMixin):
             thread.join()
         self.log.info(f"{self.__class__.__name__} joined all threads.")
 
+    def _shutdown(self):
+        """Shutdown threads and queues."""
+        self._reversed_threads = reversed(self._threads)
+        self._stop_threads()
+        self._join_threads()
+        while not self.queue.empty():
+            self.queue.get_nowait()
+        self.queue.close()
+        self.queue.join_thread()
+        self.log.info(f"{self.__class__.__name__} finished.")
+        self.log_queue.close()
+        self.log_queue.join_thread()
+
     def run(self):
         """Process tasks from the queue.
 
         This process also runs the crond thread and the Watchdog Observer
         threads.
         """
-        self.init_logger(self.log_queue)
-        self.log.debug(f"Started {self.__class__.__name__}.")
-        self.janitor = Janitor(self.log_queue, self.queue)
-        self._create_threads()  # can't do this in init.
-        self._start_threads()
-        self.run_loop = True
-        self.log.info(f"{self.__class__.__name__} ready for tasks.")
+        self._startup()
         try:
             while self.run_loop:
                 try:
@@ -189,16 +211,7 @@ class LibrarianDaemon(Process, LoggerBaseMixin):
         except KeyboardInterrupt:
             self.log.debug(f"{self.__class__.__name__} Keyboard interrupt")
         finally:
-            self._reversed_threads = reversed(self._threads)
-            self._stop_threads()
-            self._join_threads()
-            while not self.queue.empty():
-                self.queue.get_nowait()
-            self.queue.close()
-            self.queue.join_thread()
-            self.log.info(f"{self.__class__.__name__} finished.")
-            self.log_queue.close()
-            self.log_queue.join_thread()
+            self._shutdown()
 
     def stop(self):
         """Close up the librarian process."""

codex/librarian/search/optimize.py

@@ -0,0 +1,43 @@
+"""Search Index cleanup."""
+
+from time import time
+
+from django.db import connection
+from humanize import naturaldelta
+
+from codex.librarian.janitor.tasks import JanitorSearchOptimizeFinishedTask
+from codex.librarian.search.status import SearchIndexStatusTypes
+from codex.status import Status
+from codex.threads import QueuedThread
+
+_TABLE = "codex_comicfts"
+_OPTIMIZE_SQL = f"INSERT INTO {_TABLE}({_TABLE}) VALUES('optimize')"
+
+
+class OptimizeMixin(QueuedThread):
+    """Search Index optimize methods."""
+
+    def __init__(self, abort_event, *args, **kwargs):
+        """Initialize search engine."""
+        self.abort_event = abort_event
+        super().__init__(*args, **kwargs)
+
+    def optimize(self, janitor=False):
+        """Remove records not in the database from the index, trapping exceptions."""
+        start_time = time()
+        status = Status(SearchIndexStatusTypes.SEARCH_INDEX_OPTIMIZE)
+        try:
+            self.status_controller.update(status)
+            self.log.info("Optimizing search index...")
+            with connection.cursor() as cursor:
+                cursor.execute(_OPTIMIZE_SQL)
+            elapsed_time = time() - start_time
+            elapsed = naturaldelta(elapsed_time)
+            self.log.info(f"Optimized search index in {elapsed}.")
+        except Exception:
+            self.log.exception("Removing stale records:")
+        finally:
+            self.status_controller.finish(status)
+            if janitor:
+                task = JanitorSearchOptimizeFinishedTask()
+                self.librarian_queue.put(task)

codex/librarian/search/remove.py

@@ -3,56 +3,31 @@
 from time import time
 
 from humanize import naturaldelta
-from whoosh.query import Every
 
-from codex._vendor.haystack.constants import DJANGO_ID
+from codex.librarian.search.optimize import OptimizeMixin
 from codex.librarian.search.status import SearchIndexStatusTypes
-from codex.librarian.search.version import VersionMixin
-from codex.models import Comic
-from codex.search.backend import CodexSearchBackend
+from codex.models.comic import ComicFTS
 from codex.status import Status
 
 
-class RemoveMixin(VersionMixin):
+class RemoveMixin(OptimizeMixin):
     """Search Index cleanup methods."""
 
-    def _get_delete_docnums(self, backend):
-        """Get all the docunums that have pks that are *not* in the database."""
-        delete_docnums = []
-        if self.abort_event.is_set():
-            self.log.debug(
-                "Stopped search index remove stale records before it started."
-            )
-            return delete_docnums
-        database_pks = frozenset(Comic.objects.all().values_list("pk", flat=True))
-        with backend.index.refresh().searcher() as searcher:
-            results = searcher.search(Every(), scored=False)
-            for result in results:
-                index_pk = int(result.get(DJANGO_ID, 0))
-                if index_pk not in database_pks:
-                    delete_docnums.append(result.docnum)
-                if self.abort_event.is_set():
-                    self.log.debug(
-                        "Search index remove stale records will do a partial remove."
-                    )
-                    return delete_docnums
-        return delete_docnums
+    def clear_search_index(self):
+        """Clear the search index."""
+        clear_status = Status(SearchIndexStatusTypes.SEARCH_INDEX_CLEAR)
+        self.status_controller.start(clear_status)
+        ComicFTS.objects.all().delete()
+        self.status_controller.finish(clear_status)
+        self.log.info("Old search index cleared.")
 
-    def _remove_stale_records(self, backend: CodexSearchBackend | None, status):  # type: ignore
+    def _remove_stale_records(self, status):  # type: ignore
         """Remove records not in the database from the index."""
         start_time = time()
-        if not backend:
-            backend: CodexSearchBackend = self.engine.get_backend()  # type: ignore
-        if not backend.setup_complete:
-            backend.setup(False)
-
-        delete_docnums = self._get_delete_docnums(backend)
-        num_delete_docnums = len(delete_docnums)
-        count = 0
-        if num_delete_docnums:
-            status.total = num_delete_docnums
-            self.status_controller.start(status)
-            count = backend.remove_docnums(delete_docnums)
+        delete_comicfts = ComicFTS.objects.filter(comic__isnull=True)
+        status.total = len(delete_comicfts)
+        self.status_controller.update(status, notify=False)
+        count, _ = delete_comicfts.delete()
 
         # Finish
         if count:
@@ -64,17 +39,14 @@ class RemoveMixin(VersionMixin):
                 f" in {elapsed} at {cps} per second."
             )
         else:
-            self.log.debug("No stale records to remove from the search index.")
+            self.log.debug("Removed no stale records from the search index.")
 
-    def remove_stale_records(
-        self,
-        backend: CodexSearchBackend | None = None,  # type: ignore
-    ):
+    def remove_stale_records(self):
         """Remove records not in the database from the index, trapping exceptions."""
         self.abort_event.clear()
         status = Status(SearchIndexStatusTypes.SEARCH_INDEX_REMOVE)
         try:
-            self._remove_stale_records(backend, status)
+            self._remove_stale_records(status)
         except Exception:
             self.log.exception("Removing stale records:")
         finally:

codex/librarian/search/searchd.py

@@ -1,31 +1,28 @@
 """Haystack Search index updater."""
 
-from codex.librarian.search.merge import MergeMixin
 from codex.librarian.search.tasks import (
     SearchIndexClearTask,
-    SearchIndexMergeTask,
-    SearchIndexRebuildIfDBChangedTask,
+    SearchIndexOptimizeTask,
     SearchIndexRemoveStaleTask,
     SearchIndexUpdateTask,
 )
-from codex.librarian.search.update import UpdateMixin
+from codex.librarian.search.update import FTSUpdateMixin
 
 
-class SearchIndexerThread(UpdateMixin, MergeMixin):
+class SearchIndexerThread(FTSUpdateMixin):
     """A worker to handle search index update tasks."""
 
     def process_item(self, item):
         """Run the updater."""
         task = item
-        if isinstance(task, SearchIndexRebuildIfDBChangedTask):
-            self.rebuild_search_index_if_db_changed()
-        elif isinstance(task, SearchIndexUpdateTask):
-            self.update_search_index(rebuild=task.rebuild)
-        elif isinstance(task, SearchIndexMergeTask):
-            self.merge_search_index(task.optimize)
-        elif isinstance(task, SearchIndexRemoveStaleTask):
-            self.remove_stale_records()
-        elif isinstance(task, SearchIndexClearTask):
-            self.clear_search_index()
-        else:
-            self.log.warning(f"Bad task sent to search index thread: {task}")
+        match task:
+            case SearchIndexUpdateTask():
+                self.update_search_index(rebuild=task.rebuild)
+            case SearchIndexRemoveStaleTask():
+                self.remove_stale_records()
+            case SearchIndexOptimizeTask():
+                self.optimize(task.janitor)
+            case SearchIndexClearTask():
+                self.clear_search_index()
+            case _:
+                self.log.warning(f"Bad task sent to search index thread: {task}")

codex/librarian/search/status.py

@@ -8,5 +8,6 @@ class SearchIndexStatusTypes(Choices):
 
     SEARCH_INDEX_CLEAR = "SIX"
     SEARCH_INDEX_UPDATE = "SIU"
+    SEARCH_INDEX_CREATE = "SIC"
     SEARCH_INDEX_REMOVE = "SID"
-    SEARCH_INDEX_MERGE = "SIM"
+    SEARCH_INDEX_OPTIMIZE = "SIO"

codex/librarian/search/tasks.py

@@ -8,23 +8,18 @@ class SearchIndexerTask:
     """Tasks for the search indexer."""
 
 
-@dataclass
-class SearchIndexRebuildIfDBChangedTask(SearchIndexerTask):
-    """Task to check if the db is changed and schedule an update task."""
-
-
 @dataclass
 class SearchIndexUpdateTask(SearchIndexerTask):
     """Update the search index."""
 
-    rebuild: bool
+    rebuild: bool = False
 
 
 @dataclass
-class SearchIndexMergeTask(SearchIndexerTask):
-    """Merge a fragmented index."""
+class SearchIndexOptimizeTask(SearchIndexerTask):
+    """Optimize search index."""
 
-    optimize: bool = False
+    janitor: bool = False
 
 
 @dataclass