codex 1.6.3__py3-none-any.whl → 1.6.4__py3-none-any.whl

codex/librarian/importer/importer.py

@@ -42,7 +42,7 @@ class ComicImporter(MovedImporter):
             self.librarian_queue.put(LIBRARY_CHANGED_TASK)
 
             # Wait to start the search index update in case more updates are incoming.
-            until = time() + 3
+            until = time() + 1
             delayed_search_task = DelayedTasks(until, (SearchIndexUpdateTask(False),))
             self.librarian_queue.put(delayed_search_task)
         else:

codex/librarian/janitor/cleanup.py

@@ -111,7 +111,6 @@ class CleanupMixin(WorkerBaseMixin):
 
     def cleanup_custom_covers(self):
         """Clean up unused custom covers."""
-        start = time()
         covers = CustomCover.objects.only("path")
         status = Status(JanitorStatusTypes.CLEANUP_COVERS, 0, covers.count())
         delete_pks = []
@@ -128,12 +127,11 @@ class CleanupMixin(WorkerBaseMixin):
             level = logging.INFO if status.complete else logging.DEBUG
             self.log.log(level, f"Deleted {count} CustomCovers without source images.")
         finally:
-            until = start + 2
+            until = time() + 1
             self.status_controller.finish(status, until=until)
 
     def cleanup_sessions(self):
         """Delete corrupt sessions."""
-        start = time()
         status = Status(JanitorStatusTypes.CLEANUP_SESSIONS)
         try:
             self.status_controller.start(status)
@@ -152,5 +150,5 @@ class CleanupMixin(WorkerBaseMixin):
                 count, _ = bad_sessions.delete()
                 self.log.info(f"Deleted {count} corrupt sessions.")
         finally:
-            until = start + 2
+            until = time() + 1
             self.status_controller.finish(status, until=until)

codex/librarian/janitor/update.py

@@ -33,15 +33,16 @@ class UpdateMixin(WorkerBaseMixin):
 
                 self.log.info("Codex seems outdated. Trying to update.")
 
-            subprocess.run(
-                (  # noqa: S603
-                    sys.executable,
-                    "-m",
-                    "pip",
-                    "install",
-                    "--upgrade",
-                    "codex",
-                ),
+            args = (
+                sys.executable,
+                "-m",
+                "pip",
+                "install",
+                "--upgrade",
+                "codex",
+            )
+            subprocess.run(  # noqa: S603
+                args,
                 check=True,
             )
         except Exception:

codex/librarian/search/merge.py

@@ -7,7 +7,6 @@ from typing import TYPE_CHECKING
 from humanize import naturaldelta, naturalsize
 
 from codex.librarian.search.status import SearchIndexStatusTypes
-from codex.librarian.search.tasks import SearchIndexRemoveStaleTask
 from codex.librarian.search.version import VersionMixin
 from codex.settings.settings import SEARCH_INDEX_PATH
 from codex.status import Status
@@ -39,50 +38,48 @@ class MergeMixin(VersionMixin):
             size += Path(segment).stat().st_size
         return size
 
-    def merge_search_index(self, optimize=False):
+    def _merge_search_index(self, optimize, status, name):
         """Optimize search index."""
-        verb = "All" if optimize else "Small"
-        name = f"Merge {verb} Segments"
-        status = Status(SearchIndexStatusTypes.SEARCH_INDEX_MERGE, subtitle=name)
-        try:
-            statii = (
-                status,
-                Status(SearchIndexStatusTypes.SEARCH_INDEX_REMOVE),
-            )
-            self.status_controller.start_many(statii)
-            start = time()
+        self.status_controller.start(status)
+        start = time()
 
-            segments, old_num_segments = self._get_segments_and_len()
-            if self._is_index_optimized(old_num_segments):
-                return
-            self.status_controller.start(status)
-            old_size = self._get_segments_size(segments)
-            # Optimize
+        segments, old_num_segments = self._get_segments_and_len()
+        if self._is_index_optimized(old_num_segments):
+            return
+        self.status_controller.start(status)
+        old_size = self._get_segments_size(segments)
+        # Optimize
+        self.log.info(
+            f"Search index found in {old_num_segments} segments," f" merging {name}..."
+        )
+        backend: CodexSearchBackend = self.engine.get_backend()  # type: ignore
+        if optimize:
+            backend.optimize()
+        else:
+            backend.merge_small()
+
+        # Finish
+        segments, new_num_segments = self._get_segments_and_len()
+        new_size = self._get_segments_size(segments)
+        saved = naturalsize(old_size - new_size)
+        num_segments_diff = old_num_segments - new_num_segments
+        elapsed_time = time() - start
+        elapsed = naturaldelta(elapsed_time)
+        if num_segments_diff:
             self.log.info(
-                f"Search index found in {old_num_segments} segments,"
-                f" merging {name}..."
+                f"Merged {num_segments_diff} search index segments in {elapsed}."
+                f"Saved {saved}."
             )
-            backend: CodexSearchBackend = self.engine.get_backend()  # type: ignore
-            if optimize:
-                backend.optimize()
-            else:
-                backend.merge_small()
+        else:
+            self.log.info("No small search index segments found.")
 
-            # Finish
-            segments, new_num_segments = self._get_segments_and_len()
-            new_size = self._get_segments_size(segments)
-            saved = naturalsize(old_size - new_size)
-            num_segments_diff = old_num_segments - new_num_segments
-            elapsed_time = time() - start
-            elapsed = naturaldelta(elapsed_time)
-            if num_segments_diff:
-                self.log.info(
-                    f"Merged {num_segments_diff} search index segments in {elapsed}."
-                    f"Saved {saved}."
-                )
-            else:
-                self.log.info("No small search index segments found.")
-            self.librarian_queue.put(SearchIndexRemoveStaleTask())
+    def merge_search_index(self, optimize=False):
+        """Optimize search index, trapping exceptions."""
+        verb = "All" if optimize else "Small"
+        name = f"Merge {verb} Segments"
+        status = Status(SearchIndexStatusTypes.SEARCH_INDEX_MERGE, subtitle=name)
+        try:
+            self._merge_search_index(optimize, status, name)
         except Exception:
             self.log.exception("Search index merge.")
         finally:

codex/librarian/search/remove.py

@@ -38,39 +38,43 @@ class RemoveMixin(VersionMixin):
                     return delete_docnums
         return delete_docnums
 
+    def _remove_stale_records(self, backend: CodexSearchBackend | None, status):  # type: ignore
+        """Remove records not in the database from the index."""
+        start_time = time()
+        if not backend:
+            backend: CodexSearchBackend = self.engine.get_backend()  # type: ignore
+        if not backend.setup_complete:
+            backend.setup(False)
+
+        delete_docnums = self._get_delete_docnums(backend)
+        num_delete_docnums = len(delete_docnums)
+        count = 0
+        if num_delete_docnums:
+            status.total = num_delete_docnums
+            self.status_controller.start(status)
+            count = backend.remove_docnums(delete_docnums)
+
+        # Finish
+        if count:
+            elapsed_time = time() - start_time
+            elapsed = naturaldelta(elapsed_time)
+            cps = int(count / elapsed_time)
+            self.log.info(
+                f"Removed {count} stale records from the search index"
+                f" in {elapsed} at {cps} per second."
+            )
+        else:
+            self.log.debug("No stale records to remove from the search index.")
+
     def remove_stale_records(
         self,
         backend: CodexSearchBackend | None = None,  # type: ignore
     ):
-        """Remove records not in the database from the index."""
+        """Remove records not in the database from the index, trapping exceptions."""
         self.abort_event.clear()
         status = Status(SearchIndexStatusTypes.SEARCH_INDEX_REMOVE)
         try:
-            start_time = time()
-            if not backend:
-                backend: CodexSearchBackend = self.engine.get_backend()  # type: ignore
-            if not backend.setup_complete:
-                backend.setup(False)
-
-            delete_docnums = self._get_delete_docnums(backend)
-            num_delete_docnums = len(delete_docnums)
-            count = 0
-            if num_delete_docnums:
-                status.total = num_delete_docnums
-                self.status_controller.start(status)
-                count = backend.remove_docnums(delete_docnums)
-
-            # Finish
-            if count:
-                elapsed_time = time() - start_time
-                elapsed = naturaldelta(elapsed_time)
-                cps = int(count / elapsed_time)
-                self.log.info(
-                    f"Removed {count} stale records from the search index"
-                    f" in {elapsed} at {cps} per second."
-                )
-            else:
-                self.log.debug("No stale records to remove from the search index.")
+            self._remove_stale_records(backend, status)
         except Exception:
             self.log.exception("Removing stale records:")
         finally:

codex/librarian/search/update.py

@@ -301,7 +301,7 @@ class UpdateMixin(RemoveMixin):
         except Exception:
             self.log.exception("Update search index with multiprocessing")
         finally:
-            until = start_time + 1
+            until = time() + 1
             self.status_controller.finish(status, until=until)
 
     def clear_search_index(self):
@@ -313,50 +313,59 @@ class UpdateMixin(RemoveMixin):
         self.status_controller.finish(clear_status)
         self.log.info("Old search index cleared.")
 
-    def update_search_index(self, rebuild=False):
+    def _update_search_index(self, start_time, rebuild):
         """Update or Rebuild the search index."""
-        start_time = time()
-        self.abort_event.clear()
-        try:
-            any_update_in_progress = Library.objects.filter(
-                covers_only=False, update_in_progress=True
-            ).exists()
-            if any_update_in_progress:
-                self.log.debug(
-                    "Database update in progress, not updating search index yet."
-                )
-                return
+        remove_stale = False
+        any_update_in_progress = Library.objects.filter(
+            covers_only=False, update_in_progress=True
+        ).exists()
+        if any_update_in_progress:
+            self.log.debug(
+                "Database update in progress, not updating search index yet."
+            )
+            return remove_stale
 
-            if not rebuild and not self.is_search_index_uuid_match():
-                rebuild = True
+        if not rebuild and not self.is_search_index_uuid_match():
+            rebuild = True
 
-            self._init_statuses(rebuild)
+        self._init_statuses(rebuild)
 
-            # Clear
-            if rebuild:
-                self.log.info("Rebuilding search index...")
-                self.clear_search_index()
+        # Clear
+        if rebuild:
+            self.log.info("Rebuilding search index...")
+            self.clear_search_index()
 
-            # Update
-            backend: CodexSearchBackend = self.engine.get_backend()  # type: ignore
-            backend.setup(False)
-            if self.abort_event.is_set():
-                return
-            qs = self._get_queryset(backend, rebuild)
-            self._mp_update(backend, qs)
+        # Update
+        backend: CodexSearchBackend = self.engine.get_backend()  # type: ignore
+        backend.setup(False)
+        if self.abort_event.is_set():
+            self.log.debug("Abort update search index.")
+            return remove_stale
+        qs = self._get_queryset(backend, rebuild)
+        self._mp_update(backend, qs)
 
-            # Finish
-            if rebuild:
-                self.set_search_index_version()
-            else:
-                task = SearchIndexRemoveStaleTask()
-                self.librarian_queue.put(task)
+        # Finish
+        if rebuild:
+            self.set_search_index_version()
+        else:
+            remove_stale = True
 
-            elapsed_time = time() - start_time
-            elapsed = naturaldelta(elapsed_time)
-            self.log.info(f"Search index updated in {elapsed}.")
+        elapsed_time = time() - start_time
+        elapsed = naturaldelta(elapsed_time)
+        self.log.info(f"Search index updated in {elapsed}.")
+        return remove_stale
+
+    def update_search_index(self, rebuild=False):
+        """Update or Rebuild the search index."""
+        start_time = time()
+        self.abort_event.clear()
+        remove_stale = False
+        try:
+            remove_stale = self._update_search_index(start_time, rebuild)
         except Exception:
             self.log.exception("Update search index")
         finally:
-            until = start_time + 2
-            self.status_controller.finish_many(self._STATUS_FINISH_TYPES, until=until)
+            self.status_controller.finish_many(self._STATUS_FINISH_TYPES)
+            if remove_stale:
+                task = SearchIndexRemoveStaleTask()
+                self.librarian_queue.put(task)

codex/librarian/stats.py

@@ -0,0 +1,177 @@
+"""Admin Flag View."""
+
+from multiprocessing import cpu_count
+from pathlib import Path
+from platform import machine, python_version, release, system
+from types import MappingProxyType
+
+from caseconverter import snakecase
+from django.contrib.sessions.models import Session
+from django.db.models import Count
+
+from codex.logger.logging import get_logger
+from codex.models import (
+    Comic,
+    Library,
+)
+from codex.version import VERSION
+from codex.views.const import CONFIG_MODELS, METADATA_MODELS, STATS_GROUP_MODELS
+from codex.views.session import SessionView
+
+LOG = get_logger(__name__)
+_KEY_MODELS_MAP = MappingProxyType(
+    {
+        "config": CONFIG_MODELS,
+        "groups": STATS_GROUP_MODELS,
+        "metadata": METADATA_MODELS,
+    }
+)
+_DOCKERENV_PATH = Path("/.dockerenv")
+_CGROUP_PATH = Path("/proc/self/cgroup")
+_USER_STATS = MappingProxyType(
+    {
+        SessionView.BROWSER_SESSION_KEY: ("top_group", "order_by", "dynamic_covers"),
+        SessionView.READER_SESSION_KEY: (
+            "finish_on_last_page",
+            "fit_to",
+            "reading_direction",
+        ),
+    }
+)
+
+
+class CodexStats:
+    """Collect codex stats."""
+
+    def __init__(self, params=None):
+        """Specify which stats to collect. Default to all."""
+        if not params:
+            params = {}
+        self.params = params
+
+    @classmethod
+    def _is_docker(cls):
+        """Test if we're in a docker container."""
+        try:
+            return _DOCKERENV_PATH.is_file() or "docker" in _CGROUP_PATH.read_text()
+        except Exception:
+            return False
+
+    def _get_models(self, key):
+        """Get models from request params."""
+        request_model_set = self.params.get(key, {})
+        all_models = _KEY_MODELS_MAP[key]
+        if request_model_set:
+            models = []
+            for model_name in request_model_set:
+                for model in all_models:
+                    if model.__name__.lower() == model_name.lower():
+                        models.append(model)
+        else:
+            models = all_models
+        return tuple(models)
+
+    def _get_model_counts(self, key):
+        """Get database counts of each model group."""
+        models = self._get_models(key)
+        obj = {}
+        for model in models:
+            name = snakecase(model.__name__) + "_count"
+            qs = model.objects
+            if model == Library:
+                qs = qs.filter(covers_only=False)
+            obj[name] = qs.count()
+        return obj
+
+    @staticmethod
+    def _aggregate_session_key(session, session_key, session_subkeys, user_stats):
+        session_dict = session.get(session_key, {})
+        for key in session_subkeys:
+            value = session_dict.get(key)
+            if value is None:
+                continue
+            if key not in user_stats:
+                user_stats[key] = {}
+            if value not in user_stats[key]:
+                user_stats[key][value] = 0
+            user_stats[key][value] += 1
+
+    @classmethod
+    def _get_session_stats(cls):
+        """Return the number of anonymous sessions."""
+        sessions = Session.objects.all()
+        anon_session_count = 0
+        user_stats = {}
+        for encoded_session in sessions:
+            session = encoded_session.get_decoded()
+            if not session.get("_auth_user_id"):
+                anon_session_count += 1
+            for session_key, subkeys in _USER_STATS.items():
+                cls._aggregate_session_key(session, session_key, subkeys, user_stats)
+
+        return user_stats, anon_session_count
+
+    def _get_platform(self, obj):
+        """Add dict of platform information to object."""
+        platform = {
+            "docker": self._is_docker(),
+            "machine": machine(),
+            "cores": cpu_count(),
+            "system": {
+                "name": system(),
+                "release": release(),
+            },
+            "python_version": python_version(),
+            "codex_version": VERSION,
+        }
+        obj["platform"] = platform
+
+    def _get_config(self, obj):
+        """Add dict of config informaation to object."""
+        config = self._get_model_counts("config")
+        sessions, config["user_anonymous_count"] = self._get_session_stats()
+        config["user_registered_count"] = config.pop("users_count", 0)
+        config["auth_group_count"] = config.pop("groups_count", 0)
+        obj["config"] = config
+        obj["sessions"] = sessions
+
+    def _get_groups(self, obj):
+        """Add dict of groups information to object."""
+        groups = self._get_model_counts("groups")
+        obj["groups"] = groups
+
+    @staticmethod
+    def _get_file_types(obj):
+        """Query for file types."""
+        file_types = {}
+        qs = (
+            Comic.objects.values("file_type")
+            .annotate(count=Count("file_type"))
+            .order_by()
+        )
+        for query_group in qs:
+            value = query_group["file_type"]
+            name = value.lower() if value else "unknown"
+            file_types[name] = query_group["count"]
+        sorted_fts = dict(sorted(file_types.items()))
+        obj["file_types"] = sorted_fts
+
+    def _get_metadata(self, obj):
+        """Add dict of metadata counts to object."""
+        metadata = self._get_model_counts("metadata")
+        obj["metadata"] = metadata
+
+    def get(self):
+        """Construct the stats object."""
+        obj = {}
+        if not self.params or "platform" in self.params:
+            self._get_platform(obj)
+        if not self.params or "config" in self.params:
+            self._get_config(obj)
+        if not self.params or "groups" in self.params:
+            self._get_groups(obj)
+        if not self.params or "fileTypes" in self.params:
+            self._get_file_types(obj)
+        if not self.params or "metadata" in self.params:
+            self._get_metadata(obj)
+        return obj

codex/librarian/watchdog/events.py

@@ -191,8 +191,8 @@ class CodexLibraryEventHandler(CodexEventHandlerBase):
             events = self._transform_file_event(event)
 
             # Send it to the EventBatcher
-            for event in events:
-                task = WatchdogEventTask(self.library_pk, event)
+            for sub_event in events:
+                task = WatchdogEventTask(self.library_pk, sub_event)
                 self.librarian_queue.put(task)
 
             # Calls stub event dispatchers