codex 1.6.19__py3-none-any.whl → 1.7.0a0__py3-none-any.whl

codex/librarian/search/optimize.py

@@ -0,0 +1,43 @@
+"""Search Index cleanup."""
+
+from time import time
+
+from django.db import connection
+from humanize import naturaldelta
+
+from codex.librarian.janitor.tasks import JanitorSearchOptimizeFinishedTask
+from codex.librarian.search.status import SearchIndexStatusTypes
+from codex.status import Status
+from codex.threads import QueuedThread
+
+_TABLE = "codex_comicfts"
+_OPTIMIZE_SQL = f"INSERT INTO {_TABLE}({_TABLE}) VALUES('optimize')"
+
+
+class OptimizeMixin(QueuedThread):
+    """Search Index optimize methods."""
+
+    def __init__(self, abort_event, *args, **kwargs):
+        """Initialize search engine."""
+        self.abort_event = abort_event
+        super().__init__(*args, **kwargs)
+
+    def optimize(self, janitor=False):
+        """Remove records not in the database from the index, trapping exceptions."""
+        start_time = time()
+        status = Status(SearchIndexStatusTypes.SEARCH_INDEX_OPTIMIZE)
+        try:
+            self.status_controller.update(status)
+            self.log.info("Optimizing search index...")
+            with connection.cursor() as cursor:
+                cursor.execute(_OPTIMIZE_SQL)
+            elapsed_time = time() - start_time
+            elapsed = naturaldelta(elapsed_time)
+            self.log.info(f"Optimized search index in {elapsed}.")
+        except Exception:
+            self.log.exception("Removing stale records:")
+        finally:
+            self.status_controller.finish(status)
+            if janitor:
+                task = JanitorSearchOptimizeFinishedTask()
+                self.librarian_queue.put(task)

codex/librarian/search/remove.py

@@ -3,56 +3,31 @@
 from time import time
 
 from humanize import naturaldelta
-from whoosh.query import Every
 
-from codex._vendor.haystack.constants import DJANGO_ID
+from codex.librarian.search.optimize import OptimizeMixin
 from codex.librarian.search.status import SearchIndexStatusTypes
-from codex.librarian.search.version import VersionMixin
-from codex.models import Comic
-from codex.search.backend import CodexSearchBackend
+from codex.models.comic import ComicFTS
 from codex.status import Status
 
 
-class RemoveMixin(VersionMixin):
+class RemoveMixin(OptimizeMixin):
     """Search Index cleanup methods."""
 
-    def _get_delete_docnums(self, backend):
-        """Get all the docunums that have pks that are *not* in the database."""
-        delete_docnums = []
-        if self.abort_event.is_set():
-            self.log.debug(
-                "Stopped search index remove stale records before it started."
-            )
-            return delete_docnums
-        database_pks = frozenset(Comic.objects.all().values_list("pk", flat=True))
-        with backend.index.refresh().searcher() as searcher:
-            results = searcher.search(Every(), scored=False)
-            for result in results:
-                index_pk = int(result.get(DJANGO_ID, 0))
-                if index_pk not in database_pks:
-                    delete_docnums.append(result.docnum)
-                if self.abort_event.is_set():
-                    self.log.debug(
-                        "Search index remove stale records will do a partial remove."
-                    )
-                    return delete_docnums
-        return delete_docnums
+    def clear_search_index(self):
+        """Clear the search index."""
+        clear_status = Status(SearchIndexStatusTypes.SEARCH_INDEX_CLEAR)
+        self.status_controller.start(clear_status)
+        ComicFTS.objects.all().delete()
+        self.status_controller.finish(clear_status)
+        self.log.info("Old search index cleared.")
 
-    def _remove_stale_records(self, backend: CodexSearchBackend | None, status):  # type: ignore
+    def _remove_stale_records(self, status):  # type: ignore
         """Remove records not in the database from the index."""
         start_time = time()
-        if not backend:
-            backend: CodexSearchBackend = self.engine.get_backend()  # type: ignore
-        if not backend.setup_complete:
-            backend.setup(False)
-
-        delete_docnums = self._get_delete_docnums(backend)
-        num_delete_docnums = len(delete_docnums)
-        count = 0
-        if num_delete_docnums:
-            status.total = num_delete_docnums
-            self.status_controller.start(status)
-            count = backend.remove_docnums(delete_docnums)
+        delete_comicfts = ComicFTS.objects.filter(comic__isnull=True)
+        status.total = len(delete_comicfts)
+        self.status_controller.update(status, notify=False)
+        count, _ = delete_comicfts.delete()
 
         # Finish
         if count:
@@ -64,17 +39,14 @@ class RemoveMixin(VersionMixin):
                 f" in {elapsed} at {cps} per second."
             )
         else:
-            self.log.debug("No stale records to remove from the search index.")
+            self.log.debug("Removed no stale records from the search index.")
 
-    def remove_stale_records(
-        self,
-        backend: CodexSearchBackend | None = None,  # type: ignore
-    ):
+    def remove_stale_records(self):
         """Remove records not in the database from the index, trapping exceptions."""
         self.abort_event.clear()
         status = Status(SearchIndexStatusTypes.SEARCH_INDEX_REMOVE)
         try:
-            self._remove_stale_records(backend, status)
+            self._remove_stale_records(status)
         except Exception:
             self.log.exception("Removing stale records:")
         finally:

codex/librarian/search/searchd.py

@@ -1,31 +1,28 @@
 """Haystack Search index updater."""
 
-from codex.librarian.search.merge import MergeMixin
 from codex.librarian.search.tasks import (
     SearchIndexClearTask,
-    SearchIndexMergeTask,
-    SearchIndexRebuildIfDBChangedTask,
+    SearchIndexOptimizeTask,
     SearchIndexRemoveStaleTask,
     SearchIndexUpdateTask,
 )
-from codex.librarian.search.update import UpdateMixin
+from codex.librarian.search.update import FTSUpdateMixin
 
 
-class SearchIndexerThread(UpdateMixin, MergeMixin):
+class SearchIndexerThread(FTSUpdateMixin):
     """A worker to handle search index update tasks."""
 
     def process_item(self, item):
         """Run the updater."""
         task = item
-        if isinstance(task, SearchIndexRebuildIfDBChangedTask):
-            self.rebuild_search_index_if_db_changed()
-        elif isinstance(task, SearchIndexUpdateTask):
-            self.update_search_index(rebuild=task.rebuild)
-        elif isinstance(task, SearchIndexMergeTask):
-            self.merge_search_index(task.optimize)
-        elif isinstance(task, SearchIndexRemoveStaleTask):
-            self.remove_stale_records()
-        elif isinstance(task, SearchIndexClearTask):
-            self.clear_search_index()
-        else:
-            self.log.warning(f"Bad task sent to search index thread: {task}")
+        match task:
+            case SearchIndexUpdateTask():
+                self.update_search_index(rebuild=task.rebuild)
+            case SearchIndexRemoveStaleTask():
+                self.remove_stale_records()
+            case SearchIndexOptimizeTask():
+                self.optimize(task.janitor)
+            case SearchIndexClearTask():
+                self.clear_search_index()
+            case _:
+                self.log.warning(f"Bad task sent to search index thread: {task}")

codex/librarian/search/status.py

@@ -8,5 +8,6 @@ class SearchIndexStatusTypes(Choices):
 
     SEARCH_INDEX_CLEAR = "SIX"
     SEARCH_INDEX_UPDATE = "SIU"
+    SEARCH_INDEX_CREATE = "SIC"
     SEARCH_INDEX_REMOVE = "SID"
-    SEARCH_INDEX_MERGE = "SIM"
+    SEARCH_INDEX_OPTIMIZE = "SIO"

codex/librarian/search/tasks.py

@@ -8,23 +8,18 @@ class SearchIndexerTask:
     """Tasks for the search indexer."""
 
 
-@dataclass
-class SearchIndexRebuildIfDBChangedTask(SearchIndexerTask):
-    """Task to check if the db is changed and schedule an update task."""
-
-
 @dataclass
 class SearchIndexUpdateTask(SearchIndexerTask):
     """Update the search index."""
 
-    rebuild: bool
+    rebuild: bool = False
 
 
 @dataclass
-class SearchIndexMergeTask(SearchIndexerTask):
-    """Merge a fragmented index."""
+class SearchIndexOptimizeTask(SearchIndexerTask):
+    """Optimize search index."""
 
-    optimize: bool = False
+    janitor: bool = False
 
 
 @dataclass