codex 1.7.3__py3-none-any.whl → 1.7.3a0__py3-none-any.whl

codex/librarian/janitor/integrity.py

@@ -2,7 +2,6 @@
 # Uses app.get_model() because functions may also be called before the models are ready on startup.
 
 import re
-from typing import TYPE_CHECKING
 
 from django.apps import apps
 from django.db import DEFAULT_DB_ALIAS, connections
@@ -20,13 +19,6 @@ from codex.settings.settings import (
 from codex.status import Status
 from codex.worker_base import WorkerBaseMixin
 
-if TYPE_CHECKING:
-    from django.db.models.manager import BaseManager
-
-    from codex.models.base import BaseModel
-    from codex.models.comic import Comic
-
-
 REPAIR_FLAG_PATH = CONFIG_PATH / "rebuild_db"
 REBUILT_DB_PATH = DB_PATH.parent / (DB_PATH.name + ".rebuilt")
 BACKUP_DB_PATH = DB_PATH.parent / (DB_PATH.name + ".bak")
@@ -113,7 +105,7 @@ def _null_bad_fk_rels_table(table_name, bad_rows, log):
     fields = model._meta.fields
     field_names = {}
     update_objs = []
-    objs: BaseManager[BaseModel] = model.objects.filter(pk__in=bad_rows.keys())  # type: ignore[reportAssignmentType]
+    objs = model.objects.filter(pk__in=bad_rows.keys())
     update_fields = {"updated_at"}
     now = Now()
     for obj in objs:
@@ -127,7 +119,7 @@ def _null_bad_fk_rels_table(table_name, bad_rows, log):
                 field_name = field_names[fkid]
                 setattr(obj, field_name, None)
                 update_fields.add(field_name)
-            obj.updated_at = now
+            obj.updated_at = now  # type: ignore
             update_objs.append(obj)
         except Exception as exc:
             log.warning(
@@ -150,10 +142,10 @@ def _null_bad_fk_rels(bad_fk_rels, log):
     for table_name, bad_rows in bad_fk_rels.items():
         try:
             fix_comic_pks |= _null_bad_fk_rels_table(table_name, bad_rows, log)
-        except Exception:
+        except Exception as exc:
             pks = sorted(bad_rows.keys())
-            log.exception(
-                f"Unable to null {len(pks)} {table_name} rows with bad foreign keys"
+            log.error(  # noqa: TRY400
+                f"Unable to null {len(pks)} {table_name} rows with bad foreign keys - {exc}"
             )
             log.error(f"{table_name}: {pks}")  # noqa: TRY400
     return fix_comic_pks
@@ -183,8 +175,10 @@ def _delete_bad_m2m_rows(bad_m2m_rows, log):
     for table_name, ids in bad_m2m_rows.items():
         try:
             count += _delete_bad_m2m_rows_table(table_name, ids, fix_comic_pks, log)
-        except Exception:
-            log.exception(f"Could not delete bad foreign keys in table {table_name}")
+        except Exception as exc:
+            log.error(  # noqa: TRY400
+                f"Could not delete bad foreign keys in table {table_name} - {exc}"
+            )
     if count:
         log.info(f"Removed {count} bad foreign key relations.")
     return fix_comic_pks
@@ -195,9 +189,7 @@ def _mark_comics_for_update(fix_comic_pks, log):
     if not fix_comic_pks:
         return
     comic_model = apps.get_model(app_label="codex", model_name="comic")
-    outdated_comics: BaseManager[Comic] = comic_model.objects.filter(
-        pk__in=fix_comic_pks
-    ).only(  # type: ignore[reportAssignmentType]
+    outdated_comics = comic_model.objects.filter(pk__in=fix_comic_pks).only(
         "stat", "updated_at"
     )
     if not outdated_comics:
@@ -206,12 +198,12 @@ def _mark_comics_for_update(fix_comic_pks, log):
     update_comics = []
     now = Now()
     for comic in outdated_comics:
-        stat_list = comic.stat
+        stat_list = comic.stat  # type: ignore
         if not stat_list:
             continue
         stat_list[8] = 0.0
-        comic.stat = stat_list  # type: ignore[reportAttributeAccessIssue]
-        comic.updated_at = now
+        comic.stat = stat_list  # type: ignore
+        comic.updated_at = now  # type: ignore
         update_comics.append(comic)
 
     if update_comics:
@@ -235,8 +227,8 @@ def fix_foreign_keys(log=None):
     fix_comic_pks |= _delete_bad_m2m_rows(bad_m2m_rows, log)
     try:
         _mark_comics_for_update(fix_comic_pks, log)
-    except Exception:
-        LOG.exception("Could not mark comics with bad relations for update")
+    except Exception as exc:
+        LOG.warning(f"Could not mark comics with bad relations for update: {exc}")
 
 
 def _repair_extra_custom_cover_libraries(library_model, log):
@@ -285,7 +277,7 @@ def _is_integrity_ok(results):
     )
 
 
-def integrity_check(long, log=None):
+def integrity_check(long=True, log=None):
     """Run sqlite3 integrity check."""
     if not log:
         log = LOG
@@ -326,8 +318,9 @@ def fts_integrity_check(log=None):
         if results:
             # I'm not sure if this raises or puts the error in the results.
             raise ValueError(results)  # noqa: TRY301
-    except Exception:
-        log.exception("Full Text Search Index failed integrity check.")
+    except Exception as exc:
+        log.warning("Full Text Search Index failed integrity check.")
+        log.warning(exc)
     else:
         log.info("Full Text Search Index passed integrity check.")
         success = True
@@ -347,7 +340,7 @@ class IntegrityMixin(WorkerBaseMixin):
         finally:
             self.status_controller.finish(status)
 
-    def integrity_check(self, long: bool):
+    def integrity_check(self, long=False):
         """Integrity check task."""
         subtitle = "" if long else "Quick"
         status = Status(JanitorStatusTypes.INTEGRITY_CHECK, subtitle=subtitle)

codex/librarian/janitor/janitor.py

@@ -10,6 +10,7 @@ from codex.librarian.janitor.cleanup import TOTAL_NUM_FK_CLASSES, CleanupMixin
 from codex.librarian.janitor.failed_imports import UpdateFailedImportsMixin
 from codex.librarian.janitor.integrity import IntegrityMixin
 from codex.librarian.janitor.latest_version import LatestVersionMixin
+from codex.librarian.janitor.log_compress import LogCompressMixin
 from codex.librarian.janitor.status import JanitorStatusTypes
 from codex.librarian.janitor.tasks import (
     ForceUpdateAllFailedImportsTask,
@@ -20,6 +21,7 @@ from codex.librarian.janitor.tasks import (
     JanitorCleanupBookmarksTask,
     JanitorCleanupSessionsTask,
     JanitorClearStatusTask,
+    JanitorCompressOldLogs,
     JanitorForeignKeyCheck,
     JanitorFTSIntegrityCheck,
     JanitorFTSRebuildTask,
@@ -61,6 +63,7 @@ _JANITOR_STATII = (
     Status(SearchIndexStatusTypes.SEARCH_INDEX_UPDATE),
     Status(SearchIndexStatusTypes.SEARCH_INDEX_REMOVE),
     Status(SearchIndexStatusTypes.SEARCH_INDEX_OPTIMIZE),
+    Status(JanitorStatusTypes.COMPRESS_LOGS),
 )
 
 
@@ -71,6 +74,7 @@ class Janitor(
     UpdateFailedImportsMixin,
     VacuumMixin,
     IntegrityMixin,
+    LogCompressMixin,
 ):
     """Janitor inline task runner."""
 
@@ -109,7 +113,7 @@ class Janitor(
                 case JanitorVacuumTask():
                     self.vacuum_db()
                 case JanitorBackupTask():
-                    self.backup_db(show_status=True)
+                    self.backup_db()
                 case JanitorLatestVersionTask():
                     self.update_latest_version(task.force)
                 case JanitorUpdateTask():
@@ -152,9 +156,12 @@ class Janitor(
                     next_tasks = (
                         JanitorVacuumTask(),
                         JanitorBackupTask(),
+                        JanitorCompressOldLogs(),
                     )
                     for next_task in next_tasks:
                         self.librarian_queue.put(next_task)
+                case JanitorCompressOldLogs():
+                    self.log_compress()
                 case _:
                     self.log.warning(f"Janitor received unknown task {task}")
         except Exception:

codex/librarian/janitor/latest_version.py

@@ -28,7 +28,7 @@ class LatestVersionMixin(WorkerBaseMixin):
         response = requests.get(_REPO_URL, timeout=_REPO_TIMEOUT)
         return json.loads(response.text)["info"]["version"]
 
-    def update_latest_version(self, force: bool):
+    def update_latest_version(self, force=False):
         """Get the latest version from a remote repo using a cache."""
         status = Status(JanitorStatusTypes.CODEX_LATEST_VERSION)
         try:

codex/librarian/janitor/log_compress.py

@@ -0,0 +1,77 @@
+"""Rotate logs."""
+
+from dataclasses import dataclass
+from lzma import LZMAFile
+from pathlib import Path
+
+from humanize import naturalsize
+
+from codex.librarian.janitor.status import JanitorStatusTypes
+from codex.settings.settings import LOG_DIR
+from codex.status import Status
+from codex.worker_base import WorkerBaseMixin
+
+_LOG_GLOB = "codex.log.*"
+
+
+@dataclass
+class LogStats:
+    """Collect stats for compressed logs."""
+
+    count = 0
+    old_size = 0
+    new_size = 0
+
+
+class LogCompressMixin(WorkerBaseMixin):
+    """Rotate logs with janitor."""
+
+    @staticmethod
+    def _get_old_log_paths():
+        """Get old log paths to compress and sort them by reverse age."""
+        old_log_paths = {}
+        for path_str in LOG_DIR.glob(_LOG_GLOB):
+            old_path = Path(path_str)
+            try:
+                log_number = int(old_path.suffix[1:])
+                if log_number == 1:
+                    continue
+            except ValueError:
+                continue
+            old_log_paths[log_number] = old_path
+        return dict(sorted(old_log_paths.items(), reverse=True)).values()
+
+    @staticmethod
+    def _rotate_log(old_path, stats):
+        """Rotate one log file."""
+        new_path = Path(str(old_path) + ".xz")
+        if new_path.exists():
+            return
+        with (
+            LZMAFile(new_path, mode="w", preset=9) as log_archive,
+            old_path.open("r") as log_file,
+        ):
+            data = log_file.read()
+            log_archive.write(data.encode())
+        stats.count += 1
+        stats.old_size += old_path.stat().st_size
+        stats.new_size += new_path.stat().st_size
+        if new_path.exists():
+            old_path.unlink(missing_ok=True)
+
+    def log_compress(self):
+        """Compress old log files and report on savings."""
+        status = Status(JanitorStatusTypes.COMPRESS_LOGS)
+        try:
+            self.status_controller.start(status)
+            old_log_paths = self._get_old_log_paths()
+            stats = LogStats()
+            for old_path in old_log_paths:
+                self._rotate_log(old_path, stats)
+            if stats.count:
+                saved = naturalsize(stats.old_size - stats.new_size)
+                self.log.info(f"Compressed logs. Saved {saved}.")
+            else:
+                self.log.debug("No logs compressed.")
+        finally:
+            self.status_controller.finish(status)

codex/librarian/janitor/status.py

@@ -20,3 +20,4 @@ class JanitorStatusTypes(Choices):
     INTEGRITY_CHECK = "JIC"
     FTS_INTEGRITY_CHECK = "JFC"
     FTS_REBUILD = "JFR"
+    COMPRESS_LOGS = "JCL"

codex/librarian/janitor/tasks.py

@@ -107,3 +107,8 @@ class JanitorAdoptOrphanFoldersFinishedTask(JanitorTask):
 @dataclass
 class JanitorSearchOptimizeFinishedTask(JanitorTask):
     """Signals to the Janitor that the Search Optimize task is finished."""
+
+
+@dataclass
+class JanitorCompressOldLogs(JanitorTask):
+    """Compress old logs."""

codex/librarian/janitor/update.py

@@ -27,7 +27,7 @@ class UpdateMixin(WorkerBaseMixin):
         versio_latest_version = Version(latest_version)
 
         installed_versio_version = Version(VERSION)
-        if versio_latest_version.parts[1] and not installed_versio_version.parts[1]:  # type: ignore[reportIndexIssue]
+        if versio_latest_version.parts[1] and not installed_versio_version.parts[1]:  # type: ignore
             pre_blurb = "latest version is a prerelease. But installed version is not."
         else:
             result = versio_latest_version > installed_versio_version
@@ -35,7 +35,7 @@ class UpdateMixin(WorkerBaseMixin):
         self.log.debug(f"{latest_version=} > {VERSION=} = {result}{pre_blurb}")
         return result
 
-    def update_codex(self, force: bool):
+    def update_codex(self, force=False):
         """Update the package and restart everything if the version changed."""
         status = Status(JanitorStatusTypes.CODEX_UPDATE)
         try:

codex/librarian/janitor/vacuum.py

@@ -30,7 +30,7 @@ class VacuumMixin(WorkerBaseMixin):
         finally:
             self.status_controller.finish(status)
 
-    def backup_db(self, show_status: bool, backup_path=BACKUP_DB_PATH):
+    def backup_db(self, backup_path=BACKUP_DB_PATH, show_status=True):
         """Backup the database."""
         status = Status(JanitorStatusTypes.DB_BACKUP) if show_status else ""
         try:

codex/librarian/librariand.py

@@ -42,6 +42,7 @@ from codex.librarian.watchdog.tasks import (
     WatchdogSyncTask,
 )
 from codex.logger_base import LoggerBaseMixin
+from codex.settings.settings import SET_PROC_TITLE
 
 
 class LibrarianDaemon(Process, LoggerBaseMixin):
@@ -79,7 +80,7 @@ class LibrarianDaemon(Process, LoggerBaseMixin):
         startup_tasks = (
             AdoptOrphanFoldersTask(),
             WatchdogSyncTask(),
-            SearchIndexUpdateTask(rebuild=False),
+            SearchIndexUpdateTask(False),
         )
 
         for task in startup_tasks:
@@ -157,6 +158,10 @@ class LibrarianDaemon(Process, LoggerBaseMixin):
 
     def _startup(self):
         """Initialize threads."""
+        if SET_PROC_TITLE:
+            from setproctitle import setproctitle
+
+            setproctitle("Codex" + self.name)
         self.init_logger(self.log_queue)
         self.log.debug(f"Started {self.name}.")
         self.janitor = Janitor(self.log_queue, self.queue)
@@ -193,8 +198,7 @@ class LibrarianDaemon(Process, LoggerBaseMixin):
         self.log_queue.join_thread()
 
     def run(self):
-        """
-        Process tasks from the queue.
+        """Process tasks from the queue.
 
         This process also runs the crond thread and the Watchdog Observer
         threads.

codex/librarian/notifier/notifierd.py

@@ -16,8 +16,7 @@ class NotifierThread(AggregateMessageQueuedThread):
         self.cache[item.text] = item
 
     def _send_task(self, task):
-        """
-        Send a group_send message to the mulitprocess broadcast channel.
+        """Send a group_send message to the mulitprocess broadcast channel.
 
         A random consumer awaiting the broadcast channel will consume it,
         and do a group_send with it's message.

codex/librarian/search/optimize.py

@@ -22,7 +22,7 @@ class OptimizeMixin(QueuedThread):
         self.abort_event = abort_event
         super().__init__(*args, **kwargs)
 
-    def optimize(self, janitor: bool):
+    def optimize(self, janitor=False):
         """Remove records not in the database from the index, trapping exceptions."""
         start_time = time()
         status = Status(SearchIndexStatusTypes.SEARCH_INDEX_OPTIMIZE)

codex/librarian/search/remove.py

@@ -21,7 +21,7 @@ class RemoveMixin(OptimizeMixin):
         self.status_controller.finish(clear_status)
         self.log.info("Old search index cleared.")
 
-    def _remove_stale_records(self, status):
+    def _remove_stale_records(self, status):  # type: ignore
         """Remove records not in the database from the index."""
         start_time = time()
         delete_comicfts = ComicFTS.objects.filter(comic__isnull=True)

codex/librarian/search/update.py

@@ -159,7 +159,7 @@ class FTSUpdateMixin(RemoveMixin):
             obj_list.append(comicfts)
             self.log.debug(f"{len(obj_list)}/{comics.count()} entries prepped.")
 
-    def _get_comicfts_list(self, comics, create: bool):
+    def _get_comicfts_list(self, comics, create=False):
         """Create a ComicFTS object for bulk_create or bulk_update."""
         country_field = CountryField()
         language_field = LanguageField()
@@ -178,7 +178,7 @@ class FTSUpdateMixin(RemoveMixin):
             self.log.debug(f"{verb} no search entries.")
         self.status_controller.finish(status)
 
-    def _update_search_index_operate(self, comics_qs, create: bool):
+    def _update_search_index_operate(self, comics_qs, create=False):
         count = comics_qs.count()
         verb = "create" if create else "update"
         if not count:
@@ -285,7 +285,7 @@ class FTSUpdateMixin(RemoveMixin):
         elapsed = naturaldelta(elapsed_time)
         self.log.info(f"Search index updated in {elapsed}.")
 
-    def update_search_index(self, rebuild: bool):
+    def update_search_index(self, rebuild=False):
         """Update or Rebuild the search index."""
         start_time = time()
         self.abort_event.clear()

codex/librarian/telemeter/stats.py

@@ -62,12 +62,11 @@ class CodexStats:
         request_model_set = self.params.get(key, {})
         all_models = _KEY_MODELS_MAP[key]
         if request_model_set:
-            models = [
-                model
-                for model in all_models
-                for model_name in request_model_set
-                if model.__name__.lower() == model_name.lower()
-            ]
+            models = []
+            for model_name in request_model_set:
+                for model in all_models:
+                    if model.__name__.lower() == model_name.lower():
+                        models.append(model)
         else:
             models = all_models
         return tuple(models)
@@ -150,7 +149,7 @@ class CodexStats:
 
     def _add_file_types(self, obj):
         """Query for file types."""
-        if self.params and "file_types" not in self.params:
+        if self.params and "fileTypes" not in self.params:
             return
         file_types = {}
         qs = (

codex/librarian/telemeter/telemeter.py

@@ -68,6 +68,7 @@ def _send_telemetry(log):
     stats = CodexStats().get()
     data = {"stats": stats, "uuid": ts.version}
     _post_stats(log, data)
+    # log.debug("Sent anonymous stats")
     ts.save()  # update updated_at
 
 

codex/librarian/watchdog/db_snapshot.py

@@ -61,12 +61,12 @@ class CodexDatabaseSnapshot(DirectorySnapshot, LoggerBaseMixin):
     def __init__(
         self,
         path,
-        _recursive=True,  # noqa: FBT002 unused, always recursive
+        _recursive=True,  # unused, always recursive
         stat=os.stat,
         _listdir=os.listdir,  # unused for database
-        force=False,  # noqa: FBT002
+        force=False,
         log_queue=None,
-        covers_only=False,  # noqa: FBT002
+        covers_only=False,
     ):
         """Initialize like DirectorySnapshot but use a database walk."""
         self._covers_only = covers_only

codex/librarian/watchdog/dir_snapshot_diff.py

@@ -33,8 +33,7 @@ class CodexDirectorySnapshotDiff(DirectorySnapshotDiff):
         return self._get_inode(data.ref, path) == self._get_inode(data.snapshot, path)
 
     def _is_stats_equal(self, data, old_path, new_path):
-        """
-        Return if the mtime and size are equal.
+        """Return if the mtime and size are equal.
 
         For old paths in the ref and new paths in the snapshot.
         """
@@ -85,7 +84,9 @@ class CodexDirectorySnapshotDiff(DirectorySnapshotDiff):
             if not self._is_stats_equal(data, old_path, new_path):
                 data.modified.add(new_path)
 
-    def __init__(self, ref, snapshot, ignore_device: bool, inode_only_modified: bool):
+    def __init__(  # C901, PLR0912
+        self, ref, snapshot, ignore_device=False, inode_only_modified=False
+    ):
         """Create diff object."""
         self._ignore_device = ignore_device
         self._inode_only_modified = inode_only_modified

codex/librarian/watchdog/emitter.py

@@ -46,14 +46,14 @@ class DatabasePollingEmitter(EventEmitter, WorkerBaseMixin):
 
     _DIR_NOT_FOUND_TIMEOUT = 15 * 60
 
-    def __init__(  # noqa: PLR0913
+    def __init__(  # noqa PLR0913
         self,
         event_queue,
         watch,
         timeout=DEFAULT_EMITTER_TIMEOUT,
         log_queue=None,
         librarian_queue=None,
-        covers_only=False,  # noqa: FBT002
+        covers_only=False,
     ):
         """Initialize snapshot methods."""
         self.init_worker(log_queue, librarian_queue)
@@ -72,7 +72,7 @@ class DatabasePollingEmitter(EventEmitter, WorkerBaseMixin):
             # default stat and listdir params
         )
 
-    def poll(self, force: bool):
+    def poll(self, force=False):
         """Poll now, sooner than timeout."""
         self._force = force
         with self._poll_cond:
@@ -118,7 +118,7 @@ class DatabasePollingEmitter(EventEmitter, WorkerBaseMixin):
         return ok
 
     @property
-    def timeout(self) -> int | None:  # type: ignore[reportIncompatibleMethodOverride]
+    def timeout(self) -> int | None:  # type: ignore
         """Get the timeout for this emitter from its library."""
         # The timeout from the constructor, self._timeout, is thrown away in favor
         # of a dynamic timeout from the database.

codex/librarian/watchdog/event_batcherd.py

@@ -1,5 +1,4 @@
-"""
-Batch watchdog events into bulk database tasks.
+"""Batch watchdog events into bulk database tasks.
 
 Watchdog actually starts events as bulk events with the DirSnapshotDiff
 but the built-in filesystem event emitters serialize them, so the most

codex/librarian/watchdog/events.py

@@ -198,7 +198,8 @@ class CodexLibraryEventHandler(CodexEventHandlerBase):
                 task = WatchdogEventTask(self.library_pk, sub_event)
                 self.librarian_queue.put(task)
 
-            # Do not call super dispatch to call stub event dispatchers
+            # Calls stub event dispatchers
+            # super().dispatch(event)
         except Exception:
             self.log.exception(f"{self.__class__.__name__} dispatch")
 

codex/librarian/watchdog/observers.py

@@ -1,6 +1,5 @@
 """The Codex Library Watchdog Observer threads."""
 
-from watchdog.events import FileSystemEvent, FileSystemEventHandler
 from watchdog.observers import Observer
 from watchdog.observers.api import (
     DEFAULT_OBSERVER_TIMEOUT,
@@ -99,16 +98,14 @@ class UatuMixin(BaseObserver, WorkerBaseMixin):
         except Exception:
             self.log.exception(f"{self.__class__.__name__} sync library watches")
 
-    def schedule(
+    def schedule(  # type: ignore
         self,
-        event_handler: FileSystemEventHandler,
-        path: str,
-        *,
-        recursive: bool = False,
-        event_filter: list[type[FileSystemEvent]] | None = None,  # noqa: ARG002, F841, RUF100
-    ) -> ObservedWatch:
-        """
-        Override BaseObserver for Codex emitter class.
+        event_handler,
+        path,
+        recursive=False,
+        _event_filter=None,
+    ):
+        """Override BaseObserver for Codex emitter class.
 
         https://pythonhosted.org/watchdog/_modules/watchdog/observers/api.html#BaseObserver
         """
@@ -140,11 +137,9 @@ class UatuMixin(BaseObserver, WorkerBaseMixin):
 #     watchdog class structure doesn't work that way.
 
 
-class LibraryEventObserver(UatuMixin, Observer):  # type: ignore[reportGeneralTypeIssues]
+class LibraryEventObserver(UatuMixin, Observer):  # type: ignore
     """Regular observer."""
 
-    # Observer is a dynamically generated class by platform at runtime.
-
     ENABLE_FIELD = "events"
 
 
@@ -161,7 +156,7 @@ class LibraryPollingObserver(UatuMixin):
             *args, emitter_class=DatabasePollingEmitter, timeout=timeout, **kwargs
         )
 
-    def poll(self, library_pks, force: bool):
+    def poll(self, library_pks, force=False):
         """Poll each requested emitter."""
         try:
             qs = Library.objects.all()
@@ -170,7 +165,7 @@ class LibraryPollingObserver(UatuMixin):
             paths = frozenset(qs.values_list("path", flat=True))
 
             for emitter in self.emitters:
-                polling_emitter: DatabasePollingEmitter = emitter  # type: ignore[reportAssignmentType]
+                polling_emitter: DatabasePollingEmitter = emitter  # type: ignore
                 if emitter.watch.path in paths:
                     polling_emitter.poll(force)
         except Exception: