codex 1.7.15__py3-none-any.whl → 1.8.0__py3-none-any.whl

codex/librarian/watchdog/dir_snapshot_diff.py

@@ -85,8 +85,11 @@ class CodexDirectorySnapshotDiff(DirectorySnapshotDiff):
             if not self._is_stats_equal(data, old_path, new_path):
                 data.modified.add(new_path)
 
-    def __init__(self, ref, snapshot, ignore_device: bool, inode_only_modified: bool):
+    def __init__(  # pyright: ignore[reportMissingSuperCall]
+        self, ref, snapshot, *, ignore_device: bool, inode_only_modified: bool
+    ):
         """Create diff object."""
+        # Do init differently than super()
         self._ignore_device = ignore_device
         self._inode_only_modified = inode_only_modified
         data = SnapshotDiffData(
@@ -115,3 +118,18 @@ class CodexDirectorySnapshotDiff(DirectorySnapshotDiff):
 
         # Release memory
         data = None
+
+    def is_empty(self) -> bool:
+        """Any contents."""
+        return not any(
+            (
+                self._files_created,
+                self._files_deleted,
+                self._files_modified,
+                self._files_moved,
+                self._dirs_created,
+                self._dirs_deleted,
+                self._dirs_modified,
+                self._dirs_moved,
+            )
+        )

codex/librarian/watchdog/emitter.py

@@ -1,70 +1,84 @@
 """A Codex database event emitter for use by the observer."""
 
-from logging import DEBUG, WARNING
+from multiprocessing.queues import Queue
 from pathlib import Path
 from threading import Condition
 
 from django.db.models.functions import Now
 from django.utils import timezone
 from humanize import naturaldelta
-from watchdog.events import (
-    DirDeletedEvent,
-    DirModifiedEvent,
-    DirMovedEvent,
-    FileCreatedEvent,
-    FileDeletedEvent,
-    FileModifiedEvent,
-    FileMovedEvent,
+from typing_extensions import override
+from watchdog.events import FileSystemEvent
+from watchdog.observers.api import (
+    DEFAULT_EMITTER_TIMEOUT,
+    EventEmitter,
+    EventQueue,
+    ObservedWatch,
 )
-from watchdog.observers.api import DEFAULT_EMITTER_TIMEOUT, EventEmitter
 from watchdog.utils.dirsnapshot import DirectorySnapshot
 
+from codex.librarian.watchdog.const import (
+    ATTR_EVENT_MAP,
+    DIR_NOT_FOUND_TIMEOUT,
+    DOCKER_UNMOUNTED_FN,
+)
 from codex.librarian.watchdog.db_snapshot import CodexDatabaseSnapshot
 from codex.librarian.watchdog.dir_snapshot_diff import CodexDirectorySnapshotDiff
-from codex.librarian.watchdog.status import WatchdogStatusTypes
+from codex.librarian.watchdog.events import (
+    FinishPollEvent,
+    StartPollEvent,
+)
+from codex.librarian.watchdog.handlers import (
+    CodexCustomCoverEventHandler,
+    CodexLibraryEventHandler,
+)
+from codex.librarian.watchdog.status import WatchdogPollStatus
+from codex.librarian.worker import WorkerStatusMixin
 from codex.models import Library
-from codex.status import Status
-from codex.worker_base import WorkerBaseMixin
 
-_CODEX_EVENT_FILTER = [
-    FileMovedEvent,
-    FileModifiedEvent,
-    FileCreatedEvent,
-    FileDeletedEvent,
-    # FileClosedEvent,
-    # FileOpenedEvent,
-    DirMovedEvent,
-    DirModifiedEvent,
-    DirDeletedEvent,
-    # DirCreatedEvent,
-]
-_DOCKER_UNMOUNTED_FN = "DOCKER_UNMOUNTED_VOLUME"
 
+def extend_event_filter(
+    event_filter: list[type[FileSystemEvent]] | None,
+    extended_event_filter: tuple[type[FileSystemEvent], ...],
+):
+    """Initialize event filter with a some constant types."""
+    if event_filter is None:
+        event_filter = []
+    for event_type in extended_event_filter:
+        if event_type not in event_filter:
+            event_filter.append(event_type)
 
-class DatabasePollingEmitter(EventEmitter, WorkerBaseMixin):
-    """Use DatabaseSnapshots to compare against the DirectorySnapshots."""
 
-    _DIR_NOT_FOUND_TIMEOUT = 15 * 60
+class DatabasePollingEmitter(EventEmitter, WorkerStatusMixin):
+    """Dispatch an Importer Task from an diff without serializing back into the queue."""
 
     def __init__(  # noqa: PLR0913
         self,
-        event_queue,
-        watch,
-        timeout=DEFAULT_EMITTER_TIMEOUT,
-        log_queue=None,
-        librarian_queue=None,
-        covers_only=False,  # noqa: FBT002
+        event_queue: EventQueue,
+        watch: ObservedWatch,
+        *,
+        logger_,
+        librarian_queue: Queue,
+        library_id: int,
+        db_write_lock,
+        covers_only=False,
+        timeout: float = DEFAULT_EMITTER_TIMEOUT,
+        event_filter: list[type[FileSystemEvent]] | None = None,
     ):
         """Initialize snapshot methods."""
-        self.init_worker(log_queue, librarian_queue)
+        self.init_worker(logger_, librarian_queue, db_write_lock)
         self._poll_cond = Condition()
         self._force = False
+        self._manual_poll = False
         self._watch_path = Path(watch.path)
-        self._watch_path_unmounted = self._watch_path / _DOCKER_UNMOUNTED_FN
+        self._watch_path_unmounted = self._watch_path / DOCKER_UNMOUNTED_FN
         self._covers_only = covers_only
-        super().__init__(
-            event_queue, watch, timeout=timeout, event_filter=_CODEX_EVENT_FILTER
+        self._handler = (
+            CodexCustomCoverEventHandler if covers_only else CodexLibraryEventHandler
         )
+        self._library_id = library_id
+
+        super().__init__(event_queue, watch, timeout=timeout, event_filter=event_filter)
 
         self._take_dir_snapshot = lambda: DirectorySnapshot(
             self._watch.path,
@@ -72,77 +86,68 @@ class DatabasePollingEmitter(EventEmitter, WorkerBaseMixin):
             # default stat and listdir params
         )
 
-    def poll(self, force: bool):
-        """Poll now, sooner than timeout."""
-        self._force = force
-        with self._poll_cond:
-            self._poll_cond.notify()
-
-    def _take_db_snapshot(self):
-        """Get a database snapshot with optional force argument."""
-        db_snapshot = CodexDatabaseSnapshot(
-            self.watch.path,
-            self.watch.is_recursive,
-            force=self._force,
-            log_queue=self.log_queue,
-            covers_only=self._covers_only,
+    @staticmethod
+    def _get_poll_timeout(library):
+        since_last_poll = timezone.now() - library.last_poll
+        return max(
+            0,
+            library.poll_every.total_seconds() - since_last_poll.total_seconds(),
         )
-        self._force = False
-        return db_snapshot
 
-    def _is_watch_path_ok(self, library):
+    def _get_timeout(self) -> int | None:
         """Return a special timeout value if there's a problem with the watch dir."""
-        ok = False
         msg = ""
-        log_level = WARNING
+        log_level = "WARNING"
+        library = Library.objects.get(path=self.watch.path)
         if not library.poll:
             # Wait forever. Manual poll only
-            ok = None
+            log_level = "INFO"
+            msg = f"Library {self._watch_path} waiting for manual poll."
+            timeout = None
         elif not self._watch_path.is_dir():
             msg = f"Library {self._watch_path} not found. Not Polling."
+            timeout = DIR_NOT_FOUND_TIMEOUT
         elif self._watch_path_unmounted.exists():
             # Maybe overkill of caution here
             msg = f"Library {self._watch_path} looks like an unmounted docker volume. Not polling."
+            timeout = DIR_NOT_FOUND_TIMEOUT
         elif not tuple(self._watch_path.iterdir()):
             # Maybe overkill of caution here too
             msg = f"{self._watch_path} is empty. Suspect it may be unmounted. Not polling."
+            timeout = DIR_NOT_FOUND_TIMEOUT
         elif library.update_in_progress:
+            log_level = "DEBUG"
             msg = f"Library {library.path} update in progress. Not polling."
-            log_level = DEBUG
+            timeout = self._get_poll_timeout(library)
+        elif self._manual_poll:
+            log_level = "DEBUG"
+            msg = f"Manual poll requested for {library.path}"
+            self._manual_poll = False
+            timeout = 0
+        elif library.last_poll:
+            timeout = self._get_poll_timeout(library)
         else:
-            ok = True
+            msg = f"First ever poll for {library.path}"
+            log_level = "DEBUG"
+            timeout = 0
 
         if msg:
             self.log.log(log_level, msg)
 
-        return ok
+        return timeout
 
     @property
-    def timeout(self) -> int | None:  # type: ignore[reportIncompatibleMethodOverride]
+    @override
+    def timeout(self) -> int | None:  # pyright: ignore[reportIncompatibleMethodOverride]
         """Get the timeout for this emitter from its library."""
         # The timeout from the constructor, self._timeout, is thrown away in favor
         # of a dynamic timeout from the database.
-        timeout = self._timeout  # default is 1 second
         try:
-            library = Library.objects.get(path=self.watch.path)
-            ok = self._is_watch_path_ok(library)
-            if ok is None:
-                self.log.info(f"Library {self._watch_path} waiting for manual poll.")
-                return None  # None waits forever.
-            if ok is False:
-                return self._DIR_NOT_FOUND_TIMEOUT
-
-            if library.last_poll:
-                since_last_poll = timezone.now() - library.last_poll
-                timeout = max(
-                    0,
-                    library.poll_every.total_seconds()
-                    - since_last_poll.total_seconds(),
-                )
+            return self._get_timeout()
         except Exception:
             timeout = 0
             self.log.exception(f"Getting timeout for {self.watch.path}")
-        return int(timeout)
+        return timeout
 
     def _is_take_snapshot(self, timeout):
         """Determine if we should take a snapshot."""
@@ -153,15 +158,17 @@ class DatabasePollingEmitter(EventEmitter, WorkerBaseMixin):
                 )
             self._poll_cond.wait(timeout)
 
-        if not self.should_keep_running():
-            return None
+        # Zero or None are acceptable timeouts to run now.
+        return self.should_keep_running() and not self._get_timeout()
 
-        library = Library.objects.get(path=self.watch.path)
-        ok = self._is_watch_path_ok(library)
-        if ok is False:
-            self.log.warning("Not Polling.")
-            return None
-        return library
+    def _take_db_snapshot(self):
+        """Get a database snapshot with optional force argument."""
+        return CodexDatabaseSnapshot(
+            self.watch.path,
+            logger_=self.log,
+            force=self._force,
+            covers_only=self._covers_only,
+        )
 
     def _get_diff(self):
         """Take snapshots and compute the diff."""
@@ -181,68 +188,77 @@ class DatabasePollingEmitter(EventEmitter, WorkerBaseMixin):
             db_snapshot, dir_snapshot, ignore_device=True, inode_only_modified=True
         )
 
-    def _queue_events(self, diff):
+    def _queue_events(self):
         """Create and queue the events from the diff."""
-        self.log.debug(
+        diff = self._get_diff()
+        if not diff or diff.is_empty():
+            reason = f"Nothing changed for {self.watch.path} not sending anything."
+            self.log.debug(reason)
+            return
+        reason = (
             f"Poller sending unfiltered files: {len(diff.files_deleted)} "
             f"deleted, {len(diff.files_modified)} modified, "
             f"{len(diff.files_created)} created, "
-            f"{len(diff.files_moved)} moved."
-        )
-        self.log.debug(
+            f"{len(diff.files_moved)} moved. "
             f"Poller sending comic folders: {len(diff.dirs_deleted)} deleted,"
             f" {len(diff.dirs_modified)} modified,"
             f" {len(diff.dirs_moved)} moved."
         )
-        # Files.
-        # Could remove non-comics here, but handled by the EventHandler
-        for src_path in diff.files_deleted:
-            self.queue_event(FileDeletedEvent(src_path))
-        for src_path in diff.files_modified:
-            self.queue_event(FileModifiedEvent(src_path))
-        for src_path in diff.files_created:
-            self.queue_event(FileCreatedEvent(src_path))
-        for src_path, dest_path in diff.files_moved:
-            self.queue_event(FileMovedEvent(src_path, dest_path))
-
-        # Directories.
-        for src_path in diff.dirs_deleted:
-            self.queue_event(DirDeletedEvent(src_path))
-        for src_path in diff.dirs_modified:
-            self.queue_event(DirModifiedEvent(src_path))
-        # Folders are only created by comics themselves
-        # The event handler excludes DirCreatedEvent as well.
-        for src_path, dest_path in diff.dirs_moved:
-            self.queue_event(DirMovedEvent(src_path, dest_path))
+        self.log.debug(reason)
+        self.queue_event(StartPollEvent("", force=self._force))
+
+        for attr, event_class in ATTR_EVENT_MAP.items():
+            if attr.endswith("moved"):
+                for src_path, dest_path in getattr(diff, attr):
+                    self.queue_event(event_class(src_path, dest_path))
+            else:
+                for src_path in getattr(diff, attr):
+                    self.queue_event(event_class(src_path))
+
+        self.queue_event(FinishPollEvent("", force=self._force))
 
+    @override
     def queue_events(self, timeout):
-        """Queue events like PollingEmitter but always use a fresh db snapshot."""
+        """
+        Like PollingEmitter but use a fresh db snapshot and send an entire import task.
+
+        Importantly do not put ti on the actual event_queue because it will adding the
+        check_metadata_mtime flag on the import task.
+        """
         # We don't want to hit the disk continuously.
         # timeout behaves like an interval for polling emitters.
-        library = self._is_take_snapshot(timeout)
-        if not library:
+        if not self._is_take_snapshot(timeout):
+            self._force = False
+            return
+        if self.db_write_lock.locked():
+            self.log.warning("Database locked, not polling {self.watch.path}.")
             return
-        status = Status(WatchdogStatusTypes.POLL, subtitle=self.watch.path)
+        status = WatchdogPollStatus(subtitle=self.watch.path)
         try:
             self.status_controller.start(status)
             self.log.debug(f"Polling {self.watch.path}...")
-            diff = self._get_diff()
-            if not diff:
-                return
-
-            self._queue_events(diff)
-
+            self._queue_events()
+            library = Library.objects.get(path=self.watch.path)
             library.last_poll = Now()
             library.save()
-
             self.log.info(f"Polled {self.watch.path}")
         except Exception:
             self.log.exception("poll for watchdog events and queue them")
             raise
         finally:
             self.status_controller.finish(status)
+            # Reset on poll()
+            self._force = False
 
+    @override
     def on_thread_stop(self):
         """Send the poller as well."""
         with self._poll_cond:
             self._poll_cond.notify()
+
+    def poll(self, *, force: bool):
+        """Poll now, sooner than timeout."""
+        self._force = force
+        self._manual_poll = True
+        with self._poll_cond:
+            self._poll_cond.notify()

codex/librarian/watchdog/event_batcherd.py

@@ -11,95 +11,65 @@ from contextlib import suppress
 from copy import deepcopy
 from types import MappingProxyType
 
-from watchdog.events import EVENT_TYPE_MOVED
-
-from codex.librarian.importer.tasks import ImportDBDiffTask
-from codex.memory import get_mem_limit
-from codex.threads import AggregateMessageQueuedThread
+from typing_extensions import override
+from watchdog.events import EVENT_TYPE_MOVED, FileSystemEvent
+
+from codex.librarian.scribe.importer.tasks import ImportTask
+from codex.librarian.threads import AggregateMessageQueuedThread
+from codex.librarian.watchdog.const import EVENT_CLASS_DIFF_ALL_MAP
+from codex.librarian.watchdog.events import (
+    EVENT_TYPE_FINISH_POLL,
+    EVENT_TYPE_START_POLL,
+)
+from codex.librarian.watchdog.memory import get_mem_limit
+
+_IMPORT_TASK_PARAMS = MappingProxyType(
+    {
+        "dirs_moved": {},
+        "dirs_modified": set(),
+        "dirs_deleted": set(),
+        "files_moved": {},
+        "files_modified": set(),
+        "files_created": set(),
+        "files_deleted": set(),
+        "covers_moved": {},
+        "covers_modified": set(),
+        "covers_created": set(),
+        "covers_deleted": set(),
+    }
+)
 
 
 class WatchdogEventBatcherThread(AggregateMessageQueuedThread):
     """Batch watchdog events into bulk database tasks."""
 
-    DBDIFF_TASK_PARAMS = MappingProxyType(
-        {
-            "library_id": None,
-            "dirs_moved": {},
-            "dirs_modified": set(),
-            "dirs_deleted": set(),
-            "files_moved": {},
-            "files_modified": set(),
-            "files_created": set(),
-            "files_deleted": set(),
-            "covers_moved": {},
-            "covers_modified": set(),
-            "covers_created": set(),
-            "covers_deleted": set(),
-        }
-    )
-    MAX_DELAY = 60
+    MAX_DELAY = 60.0
     MAX_ITEMS_PER_GB = 50000
 
-    def __init__(self, *args, **kwargs):
-        """Set the total items for limiting db ops per batch."""
-        super().__init__(*args, **kwargs)
-        self._total_items = 0
-        mem_limit_gb = get_mem_limit("g")
-        self.max_items = int(self.MAX_ITEMS_PER_GB * mem_limit_gb)
-
-    def _ensure_library_args(self, library_id):
-        if library_id in self.cache:
-            return
-        args = deepcopy(dict(self.DBDIFF_TASK_PARAMS))
-        args["library_id"] = library_id
-        self.cache[library_id] = args
-
-    def _args_field_by_event(self, library_id, event):
-        """Translate event class names into field names."""
-        self._ensure_library_args(library_id)
-
-        prefix = (
-            "dir"
-            if event.is_directory
-            else "cover"
-            if getattr(event, "is_cover", False)
-            else "file"
-        )
-        field = f"{prefix}s_{event.event_type}"
-
-        return self.cache[library_id].get(field)
-
-    def aggregate_items(self, item):
-        """Aggregate events into cache by library."""
-        event = item.event
-        args_field = self._args_field_by_event(item.library_id, event)
-        if args_field is None:
-            self.log.debug(f"Unhandled event, not batching: {event}")
-            return
-        if event.event_type == EVENT_TYPE_MOVED:
-            args_field[event.src_path] = event.dest_path
-        else:
-            args_field.add(event.src_path)
-        self._total_items += 1
-        if self._total_items > self.max_items:
-            self.log.info(
-                "Event batcher hit size limit."
-                f" Sending batch of {self._total_items} to importer."
-            )
-            # Seall items
-            self.timed_out()
+    @staticmethod
+    def create_import_task_args(library_id: int) -> dict:
+        """Create import task args."""
+        args = deepcopy(dict(_IMPORT_TASK_PARAMS))
+        args["library_id"] = library_id  # pyright: ignore[reportArgumentType]
+        return args
+
+    @staticmethod
+    def key_for_event(event: FileSystemEvent) -> str:
+        """Return the args field name for the event."""
+        return EVENT_CLASS_DIFF_ALL_MAP[type(event)]
+
+    @staticmethod
+    def _remove_paths(args, deleted_key: str, moved_key: str):
+        for src_path in args[deleted_key]:
+            with suppress(KeyError):
+                del args[moved_key][src_path]
 
-    def _deduplicate_events(self, library_id):
+    @classmethod
+    def deduplicate_events(cls, args: dict):
         """Prune different event types on the same paths."""
-        args = self.cache[library_id]
-
         # deleted
-        for src_path in args["dirs_deleted"]:
-            with suppress(KeyError):
-                del args["dirs_moved"][src_path]
-        for src_path in args["files_deleted"]:
-            with suppress(KeyError):
-                del args["files_moved"][src_path]
+        cls._remove_paths(args, "dirs_deleted", "dirs_moved")
+        cls._remove_paths(args, "files_deleted", "files_moved")
 
         # created
         args["files_created"] -= args["files_deleted"]
@@ -117,20 +87,96 @@ class WatchdogEventBatcherThread(AggregateMessageQueuedThread):
         args["covers_modified"] -= args["covers_deleted"]
         args["covers_modified"] -= args["covers_created"]
 
+    def __init__(self, *args, **kwargs):
+        """Set the total items for limiting db ops per batch."""
+        super().__init__(*args, **kwargs)
+        self._total_items = 0
+        mem_limit_gb = get_mem_limit("g")
+        self.max_items = int(self.MAX_ITEMS_PER_GB * mem_limit_gb)
+
+    def _args_field_by_event(self, library_id: int, event: FileSystemEvent):
+        """Translate event class names into field names."""
+        if library_id not in self.cache:
+            self.cache[library_id] = self.create_import_task_args(library_id)
+        if key := EVENT_CLASS_DIFF_ALL_MAP.get(type(event)):
+            args_field = self.cache[library_id].get(key)
+        else:
+            reason = f"Unhandled event, not batching: {event}"
+            raise ValueError(reason)
+        return args_field
+
+    @override
+    def aggregate_items(self, item):
+        """Aggregate events into cache by library."""
+        event = item.event
+        try:
+            args_field = self._args_field_by_event(item.library_id, event)
+            if event.event_type == EVENT_TYPE_MOVED:
+                args_field[event.src_path] = event.dest_path
+            else:
+                args_field.add(event.src_path)
+            self._total_items += 1
+        except ValueError as exc:
+            self.log.debug(exc)
+        if self._total_items > self.max_items:
+            reason = (
+                "Event batcher hit size limit."
+                f" Sending batch of {self._total_items} to importer."
+            )
+            self.log.info(reason)
+            # Send all items
+            self.timed_out()
+
+    def _set_check_metadata_mtime(self, item):
+        pk = item.library_id
+        if pk not in self.cache:
+            self.cache[pk] = self.create_import_task_args(pk)
+        self.cache[pk]["check_metadata_mtime"] = not item.event.force
+
+    def _start_poll(self, item):
+        self.set_last_send()
+        self._set_check_metadata_mtime(item)
+
+    def _finish_poll(self, item):
+        self._set_check_metadata_mtime(item)
+        self.send_all_items()
+
+    @override
+    def process_item(self, item):
+        event_type = item.event.event_type
+        if event_type == EVENT_TYPE_START_POLL:
+            self._start_poll(item)
+        elif event_type == EVENT_TYPE_FINISH_POLL:
+            self._finish_poll(item)
+        else:
+            super().process_item(item)
+
+    def _subtract_args_items(self, args):
+        total = 0
+        for value in args.values():
+            with suppress(TypeError):
+                total += len(value)
+        self._total_items = max(0, self._total_items - total)
+
     def _create_task(self, library_id):
         """Create a task from cached aggregated message data."""
-        self._deduplicate_events(library_id)
-        args = self.cache[library_id]
-        return ImportDBDiffTask(**args)
+        args = self.cache.pop(library_id)
+        self._subtract_args_items(args)
+        self.deduplicate_events(args)
+        return ImportTask(**args)  # ty: ignore[missing-argument]
+
+    def _send_import_task(self, library_id: int):
+        task = self._create_task(library_id)
+        if task.total():
+            self.librarian_queue.put(task)
+        else:
+            self.log.debug("Empty task after filtering. Not sending to importer.")
 
+    @override
     def send_all_items(self):
         """Send all tasks to library queue and reset events cache."""
-        library_ids = set()
-        for library_id in self.cache:
-            task = self._create_task(library_id)
-            self.librarian_queue.put(task)
-            library_ids.add(library_id)
+        for library_id in tuple(self.cache):
+            self._send_import_task(library_id)
 
         # reset the event aggregates
-        self.cleanup_cache(library_ids)
         self._total_items = 0