codex 1.7.15__py3-none-any.whl → 1.8.0__py3-none-any.whl

codex/librarian/watchdog/observers.py

@@ -1,34 +1,35 @@
 """The Codex Library Watchdog Observer threads."""
 
-from watchdog.events import FileSystemEvent, FileSystemEventHandler
+from multiprocessing.queues import Queue
+
+from typing_extensions import override
+from watchdog.events import (
+    FileSystemEvent,
+    FileSystemEventHandler,
+)
 from watchdog.observers import Observer
 from watchdog.observers.api import (
     DEFAULT_OBSERVER_TIMEOUT,
     BaseObserver,
+    EventDispatcher,
     ObservedWatch,
 )
 
+from codex.librarian.watchdog.const import EVENT_FILTER, POLLING_EVENT_FILTER
 from codex.librarian.watchdog.emitter import DatabasePollingEmitter
-from codex.librarian.watchdog.events import (
+from codex.librarian.watchdog.handlers import (
     CodexCustomCoverEventHandler,
     CodexLibraryEventHandler,
 )
+from codex.librarian.worker import WorkerMixin
 from codex.models import Library
-from codex.worker_base import WorkerBaseMixin
 
 
-class UatuMixin(BaseObserver, WorkerBaseMixin):
-    """Watch over librarys from the blue area of the moon."""
+class UatuObserver(WorkerMixin, BaseObserver):
+    """Watch over libraries from the blue area of the moon."""
 
-    ENABLE_FIELD = ""
-    ALWAYS_WATCH = False
-
-    def __init__(self, *args, **kwargs):
-        """Initialize queues."""
-        log_queue = kwargs.pop("log_queue")
-        librarian_queue = kwargs.pop("librarian_queue")
-        self.init_worker(log_queue, librarian_queue)
-        super().__init__(*args, **kwargs)
+    ENABLE_FIELD: str = ""
+    ALWAYS_WATCH: bool = False
 
     def _get_watch(self, path):
         """Find the watch by path."""
@@ -60,9 +61,10 @@ class UatuMixin(BaseObserver, WorkerBaseMixin):
             else CodexLibraryEventHandler
         )
         handler = handler_class(
-            library,
+            library.pk,
+            logger_=self.log,
             librarian_queue=self.librarian_queue,
-            log_queue=self.log_queue,
+            db_write_lock=self.db_write_lock,
         )
         self.schedule(handler, library.path, recursive=True)
         self.log.info(f"Started {watching_log}")
@@ -75,10 +77,11 @@ class UatuMixin(BaseObserver, WorkerBaseMixin):
                 orphan_watches.add(watch)
         for watch in orphan_watches:
             self.unschedule(watch)
-            self.log.info(
+            reason = (
                 f"Stopped watching orphaned library {watch.path} "
                 f"with {self.ENABLE_FIELD}"
             )
+            self.log.info(reason)
 
     def sync_library_watches(self):
         """Watch or unwatch all libraries according to the db."""
@@ -94,74 +97,104 @@ class UatuMixin(BaseObserver, WorkerBaseMixin):
                         f"Could not find {library.path} to watch. May be unmounted."
                     )
                 except Exception:
-                    self.log.exception(f"sync library watch for {library.path}")
+                    self.log.exception(f"Sync library watch for {library.path}")
             self._unschedule_orphan_watches(library_paths)
         except Exception:
             self.log.exception(f"{self.__class__.__name__} sync library watches")
 
+    def _add_emitter_for_watch(
+        self, event_handler: FileSystemEventHandler, watch: ObservedWatch
+    ):
+        """If we don't have an emitter for this watch already, create it."""
+        covers_only = isinstance(event_handler, CodexCustomCoverEventHandler)
+        library_id = Library.objects.get(path=watch.path).pk
+        emitter = DatabasePollingEmitter(
+            event_queue=self.event_queue,
+            watch=watch,
+            timeout=self.timeout,
+            logger_=self.log,
+            librarian_queue=self.librarian_queue,
+            covers_only=covers_only,
+            library_id=library_id,
+            db_write_lock=self.db_write_lock,
+            event_filter=list(POLLING_EVENT_FILTER),
+        )
+        self._add_emitter(emitter)
+        if self.is_alive():
+            emitter.start()
+
+    @override
     def schedule(
         self,
         event_handler: FileSystemEventHandler,
         path: str,
         *,
-        recursive: bool = False,
-        event_filter: list[type[FileSystemEvent]] | None = None,  # noqa: ARG002, F841, RUF100
+        event_filter: list[type[FileSystemEvent]] | None = None,
+        **kwargs,
     ) -> ObservedWatch:
-        """
-        Override BaseObserver for Codex emitter class.
-
-        https://pythonhosted.org/watchdog/_modules/watchdog/observers/api.html#BaseObserver
-        """
+        """Override BaseObserver for Codex emitter class."""
         if self._emitter_class != DatabasePollingEmitter:
-            return super().schedule(event_handler, path, recursive=recursive)
+            return super().schedule(
+                event_handler, path, event_filter=list(POLLING_EVENT_FILTER), **kwargs
+            )
         with self._lock:
-            watch = ObservedWatch(path, recursive=recursive)
+            watch = ObservedWatch(path, event_filter=list(EVENT_FILTER), **kwargs)  # ty: ignore[missing-argument]
             self._add_handler_for_watch(event_handler, watch)
-
-            # If we don't have an emitter for this watch already, create it.
             if self._emitter_for_watch.get(watch) is None:
-                covers_only = isinstance(event_handler, CodexCustomCoverEventHandler)
-                emitter = DatabasePollingEmitter(
-                    event_queue=self.event_queue,
-                    watch=watch,
-                    timeout=self.timeout,
-                    log_queue=self.log_queue,
-                    librarian_queue=self.librarian_queue,
-                    covers_only=covers_only,
-                )
-                self._add_emitter(emitter)
-                if self.is_alive():
-                    emitter.start()
+                self._add_emitter_for_watch(event_handler, watch)
             self._watches.add(watch)
         return watch
 
 
-# It would be best for Codex to have one observer with multiple emitters, but the
-#     watchdog class structure doesn't work that way.
+#########################
+# Observer Architecture #
+#########################
+# It would be better if could have one observer per path with multiple emitters, but the
+# watchdog Observers key ObservedWatches on paths with one emitter each.
+# But Observers have only one emitter_class and I can't override the FileSystem Emitters
+# because they're generated per platform and environment. So the only overridable
+# Emitter is my own DatabasePollingEmitter.
 
 
-class LibraryEventObserver(UatuMixin, Observer):  # type: ignore[reportGeneralTypeIssues]
+class LibraryEventObserver(UatuObserver, Observer):  # pyright: ignore[reportGeneralTypeIssues, reportUntypedBaseClass], # ty: ignore[invalid-base]
     """Regular observer."""
 
     # Observer is a dynamically generated class by platform at runtime.
+    # Which causes the above static typing warning
 
-    ENABLE_FIELD = "events"
+    ENABLE_FIELD: str = "events"
+
+    def __init__(self, logger_, librarian_queue: Queue, db_write_lock, *args, **kwargs):
+        """Initialize queues."""
+        if db_write_lock is None:
+            reason = "db_write_lock argument must be a Lock."
+            raise ValueError(reason)
+        self.init_worker(logger_, librarian_queue, db_write_lock)
+        super().__init__(*args, **kwargs)
 
 
-class LibraryPollingObserver(UatuMixin):
+class LibraryPollingObserver(UatuObserver):
     """An Observer that polls using the DatabasePollingEmitter."""
 
-    ENABLE_FIELD = "poll"
-    ALWAYS_WATCH = True  # In the emitter, timeout=None for forever
-    _SHUTDOWN_EVENT = (None, None)
+    ENABLE_FIELD: str = "poll"
+    ALWAYS_WATCH: bool = True  # In the emitter, timeout=None for forever
 
-    def __init__(self, *args, timeout=DEFAULT_OBSERVER_TIMEOUT, **kwargs):
+    def __init__(
+        self,
+        logger_,
+        librarian_queue: Queue,
+        db_write_lock,
+        *args,
+        timeout: float = DEFAULT_OBSERVER_TIMEOUT,
+        **kwargs,
+    ):
         """Use the DatabasePollingEmitter."""
+        self.init_worker(logger_, librarian_queue, db_write_lock)
         super().__init__(
-            *args, emitter_class=DatabasePollingEmitter, timeout=timeout, **kwargs
+            emitter_class=DatabasePollingEmitter, timeout=timeout, **kwargs
         )
 
-    def poll(self, library_pks, force: bool):
+    def poll(self, library_pks, *, force: bool):
         """Poll each requested emitter."""
         try:
             qs = Library.objects.all()
@@ -170,19 +203,20 @@ class LibraryPollingObserver(UatuMixin):
             paths = frozenset(qs.values_list("path", flat=True))
 
             for emitter in self.emitters:
-                polling_emitter: DatabasePollingEmitter = emitter  # type: ignore[reportAssignmentType]
+                polling_emitter: DatabasePollingEmitter = emitter  # pyright: ignore[reportAssignmentType]
                 if emitter.watch.path in paths:
-                    polling_emitter.poll(force)
+                    polling_emitter.poll(force=force)
         except Exception:
             self.log.exception(
                 f"{self.__class__.__name__}.poll({library_pks}, {force})"
             )
 
+    @override
     def on_thread_stop(self):
         """Put a dummy event on the queue that blocks forever."""
         # The global timeout is None because the emitters have their own
         # per watch timeout. This makes self.dispatch_events() block
         # forever on the queue. Sending it an event makes it check the
         # shutdown event next.
-        self.event_queue.put(self._SHUTDOWN_EVENT)
+        self.event_queue.put(EventDispatcher.stop_event)  # self._SHUTDOWN_EVENT)
         super().on_thread_stop()

codex/librarian/watchdog/status.py

@@ -1,9 +1,22 @@
-"""Watchdog Status Types."""
+"""Watchdog Statii."""
 
-from django.db.models import TextChoices
+from abc import ABC
 
+from codex.librarian.status import Status
 
-class WatchdogStatusTypes(TextChoices):
-    """Watchdog Status Types."""
 
-    POLL = "WPO"
+class WatchdogStatus(Status, ABC):
+    """Watchdog Statii."""
+
+
+class WatchdogPollStatus(WatchdogStatus):
+    """Watchdog Poll Status."""
+
+    CODE = "WPO"
+    VERB = "Poll"
+    _verbed = "Polled"
+    ITEM_NAME = "library"
+    SINGLE = True
+
+
+WATCHDOG_STATII = (WatchdogPollStatus,)

codex/librarian/watchdog/tasks.py

@@ -1,12 +1,13 @@
 """Watchdog Tasks."""
 
+from abc import ABC
 from dataclasses import dataclass
 
 from watchdog.events import FileSystemEvent
 
 
 @dataclass
-class WatchdogTask:
+class WatchdogTask(ABC):  # noqa: B024
     """Watchdog tasks."""
 
 

codex/librarian/worker.py

@@ -0,0 +1,32 @@
+"""Mixin because watchdog Observers already inherit from Thread.."""
+
+from multiprocessing.queues import Queue
+
+from loguru._logger import Logger
+from typing_extensions import override
+
+from codex.librarian.status_controller import StatusController
+
+
+class WorkerMixin:
+    """Mixin for common thread attributes."""
+
+    def init_worker(self, /, logger_: Logger, librarian_queue: Queue, db_write_lock):
+        """Initialize queues."""
+        if not all((logger_, librarian_queue, db_write_lock)):
+            reason = f"{logger_=}, {librarian_queue=}, and {db_write_lock=} must be passed in."
+            raise ValueError(reason)
+        self.log = logger_  # pyright: ignore[reportUninitializedInstanceVariable]
+        self.librarian_queue = librarian_queue  # pyright: ignore[reportUninitializedInstanceVariable]
+        self.db_write_lock = db_write_lock  # pyright: ignore[reportUninitializedInstanceVariable]
+
+
+class WorkerStatusMixin(WorkerMixin):
+    """Worker mixin also sets up status controller."""
+
+    @override
+    def init_worker(self, /, logger_, librarian_queue: Queue, db_write_lock):
+        super().init_worker(logger_, librarian_queue, db_write_lock)
+        self.status_controller = StatusController(  # pyright: ignore[reportUninitializedInstanceVariable]
+            logger_, librarian_queue
+        )

codex/middleware.py

@@ -4,11 +4,9 @@ from time import time
 
 from django.db import connection
 from django.utils import timezone
+from loguru import logger
 
-from codex.logger.logger import get_logger
-from codex.settings.settings import LOG_RESPONSE_TIME, SLOW_QUERY_LIMIT
-
-LOG = get_logger(__name__)
+from codex.settings import LOG_RESPONSE_TIME, SLOW_QUERY_LIMIT
 
 
 class TimezoneMiddleware:
@@ -46,9 +44,9 @@ class LogResponseTimeMiddleware:
         if is_slow or LOG_RESPONSE_TIME:
             msg = f"{response_time}s {request.build_absolute_uri()}"
             if is_slow:
-                LOG.warning(msg)
+                logger.warning(msg)
             else:
-                LOG.debug(msg)
+                logger.trace(msg)
         return response
 
     def _log_query_times(self):
@@ -58,9 +56,9 @@ class LogResponseTimeMiddleware:
             if LOG_RESPONSE_TIME or is_slow:
                 msg = f"{query['time']}s {query['sql']}"
                 if is_slow:
-                    LOG.warning(msg)
+                    logger.warning(msg)
                 else:
-                    LOG.debug(msg)
+                    logger.trace(msg)
 
     def __call__(self, request):
         """Call request."""

codex/migrations/0011_library_groups_and_metadata_changes.py

@@ -17,10 +17,11 @@ def critical_rating_to_decimal(apps, _schema_editor):
             comic.critical_rating_decimal = Decimal(comic.critical_rating)
             update_comics.append(comic)
         except Exception as err:
-            print(
+            reason = (
                 f"WARNING: comic {comic.pk} {comic.path} failed converting "
                 f"critical_rating to Decimal {err}"
             )
+            print(reason)
     comic_model.objects.bulk_update(update_comics, ("critical_rating_decimal",))
 
 

codex/migrations/0013_int_issue_count_longer_charfields.py

@@ -17,10 +17,11 @@ def cast_issue_count(apps, _schema_editor):
             volume.issue_count = round(volume.issue_count_decimal)
             update_volumes.append(volume)
         except Exception:
-            print(
+            reason = (
                 f"unable to cast volume {volume.name} "
                 f"issue_count {volume.old_issue_count} to int"
             )
+            print(reason)
 
     volume_model.objects.bulk_update(update_volumes, ("issue_count",))
 

codex/migrations/0023_rename_credit_creator_and_more.py

@@ -12,10 +12,11 @@ def prepare_librarianstatus(apps, _schema_editor):
     ls_model = apps.get_model("codex", "librarianstatus")
     ls_model.objects.all().delete()
     with connection.cursor() as cursor:
-        cursor.execute(
+        sql = (
             "UPDATE sqlite_sequence SET seq = 0"
             ' WHERE sqlite_sequence.name = "codex_librarianstatus"'
         )
+        cursor.execute(sql)
 
 
 def prepare_bookmarks(apps, _schema_editor):

codex/migrations/0026_comicbox_1.py

@@ -4,7 +4,7 @@ from types import MappingProxyType
 
 import django.db.models.deletion
 import pycountry
-from comicbox.identifiers import parse_identifier
+from comicbox.identifiers.urns import parse_string_identifier
 from django.db import migrations, models
 
 MIGRATE_GTIN = True
@@ -132,16 +132,16 @@ def _migrate_gtin_to_ids_scan(comics):
     comic_identifiers = {}
     for comic in comics:
         try:
-            identifier_type, code = parse_identifier(comic.gtin)
+            nid, _, nss = parse_string_identifier(comic.gtin)
         except Exception:
-            identifier_type = None
-            code = None
-        if not code:
+            nid = None
+            nss = None
+        if not nss:
             continue
-        if identifier_type not in identifiers:
-            identifiers[identifier_type] = set()
-        identifiers[identifier_type].add(code)
-        comic_identifiers[comic] = (identifier_type, code)
+        if nid not in identifiers:
+            identifiers[nid] = set()
+        identifiers[nid].add(nss)
+        comic_identifiers[comic] = (nid, nss)
     return identifiers, comic_identifiers