codex 1.7.15__py3-none-any.whl → 1.8.0a2__py3-none-any.whl

codex/librarian/watchdog/events.py

@@ -1,35 +1,18 @@
-"""Watchdog Event Handlers."""
+"""Codex Filesystem Events."""
 
-import re
-from os import fsdecode
-from pathlib import Path
-from types import MappingProxyType
+from abc import ABC
+from dataclasses import dataclass
 
-from comicbox.box import Comicbox
 from watchdog.events import (
-    EVENT_TYPE_CLOSED,
-    EVENT_TYPE_CLOSED_NO_WRITE,
-    EVENT_TYPE_CREATED,
-    EVENT_TYPE_DELETED,
-    EVENT_TYPE_MODIFIED,
-    EVENT_TYPE_MOVED,
-    EVENT_TYPE_OPENED,
     FileCreatedEvent,
     FileDeletedEvent,
     FileModifiedEvent,
     FileMovedEvent,
-    FileSystemEventHandler,
+    FileSystemEvent,
 )
 
-from codex.librarian.watchdog.tasks import WatchdogEventTask
-from codex.logger.logger import get_logger
-from codex.logger_base import LoggerBaseMixin
-from codex.models import CustomCover
-from codex.settings.settings import CUSTOM_COVERS_DIR, CUSTOM_COVERS_GROUP_DIRS
-
-LOG = get_logger(__name__)
-_IMAGE_EXTS = frozenset({"jpg", "jpeg", "webp", "png", "gif", "bmp"})
-_GROUP_COVERS_DIRS = frozenset(CUSTOM_COVERS_GROUP_DIRS)
+EVENT_TYPE_START_POLL = "start_poll"
+EVENT_TYPE_FINISH_POLL = "finish_poll"
 
 
 class CoverMovedEvent(FileMovedEvent):
@@ -60,193 +43,21 @@ class CoverDeletedEvent(FileDeletedEvent):
     is_synthetic = True
 
 
-COVERS_EVENT_TYPE_MAP = MappingProxyType(
-    {
-        EVENT_TYPE_MODIFIED: CoverModifiedEvent,
-        EVENT_TYPE_CREATED: CoverCreatedEvent,
-        EVENT_TYPE_DELETED: CoverDeletedEvent,
-    }
-)
+@dataclass(unsafe_hash=True)
+class CodexPollEvent(FileSystemEvent, ABC):
+    """Coex poll event."""
+
+    force: bool = False
+    is_synthetic = True
+
+
+class StartPollEvent(CodexPollEvent):
+    """Start Import Task."""
+
+    event_type = EVENT_TYPE_START_POLL
+
 
+class FinishPollEvent(CodexPollEvent):
+    """Finish Import Task."""
 
-class CodexEventHandlerBase(FileSystemEventHandler, LoggerBaseMixin):
-    """Base class for Codex Event Handlers."""
-
-    IGNORED_EVENTS = frozenset(
-        {EVENT_TYPE_CLOSED, EVENT_TYPE_CLOSED_NO_WRITE, EVENT_TYPE_OPENED}
-    )
-
-    def __init__(self, library, *args, **kwargs):
-        """Let us send along he library id."""
-        self.library_pk = library.pk
-        self.librarian_queue = kwargs.pop("librarian_queue")
-        log_queue = kwargs.pop("log_queue")
-        self.init_logger(log_queue)
-        super().__init__(*args, **kwargs)
-
-    @staticmethod
-    def _match_image_suffix(path: Path) -> bool:
-        return path and len(path.suffix) > 1 and path.suffix[1:] in _IMAGE_EXTS
-
-
-class CodexLibraryEventHandler(CodexEventHandlerBase):
-    """Handle watchdog events for comics in a library."""
-
-    def _set_comic_matcher(self):
-        comic_regex = r"\.(cb[zt"
-        unsupported = []
-        if Comicbox.is_unrar_supported():
-            comic_regex += r"r"
-        else:
-            unsupported.append("CBR")
-        comic_regex += r"]"
-
-        if Comicbox.is_pdf_supported():
-            comic_regex += r"|pdf"
-        else:
-            unsupported.append("PDF")
-        comic_regex += ")$"
-        self._comic_matcher = re.compile(comic_regex, re.IGNORECASE)
-        if unsupported:
-            un_str = ", ".join(unsupported)
-            LOG.warning(f"Cannot detect or read from {un_str} archives")
-
-    def __init__(self, *args, **kwargs):
-        """Let us send along he library id."""
-        self._set_comic_matcher()
-        super().__init__(*args, **kwargs)
-
-    def _match_comic_suffix(self, path):
-        """Match a supported comic suffix."""
-        if not path:
-            return False
-        # We don't care about general suffixes. Only length four.
-        suffix = path[-4:]
-        suffix = fsdecode(suffix)
-        return self._comic_matcher.match(suffix) is not None
-
-    @classmethod
-    def _match_folder_cover(cls, path_str: str) -> bool:
-        if not path_str:
-            return False
-        path = Path(path_str)
-        return path.stem == CustomCover.FOLDER_COVER_STEM and cls._match_image_suffix(
-            path
-        )
-
-    def _transform_file_move_event(
-        self,
-        event,
-        events,
-        source_match_comic,
-    ):
-        """Create file events for file moves."""
-        dest_match_comic = self._match_comic_suffix(event.dest_path)
-        if not source_match_comic and dest_match_comic:
-            # Moved from an ignored file extension into a comic type,
-            # so create a new comic.
-            events.append(FileCreatedEvent(event.dest_path))
-        elif source_match_comic and not dest_match_comic:
-            # moved into something that's not a comic name so delete
-            events.append(FileDeletedEvent(event.src_path))
-        elif source_match_comic and dest_match_comic:
-            events.append(event)
-
-    def _transform_file_move_event_to_cover_event(self, event, events):
-        """Create cover events for file moves."""
-        source_match_cover = self._match_folder_cover(event.src_path)
-        dest_match_cover = self._match_folder_cover(event.dest_path)
-        if source_match_cover and not dest_match_cover:
-            events.append(CoverDeletedEvent(event.src_path))
-        elif not source_match_cover and dest_match_cover:
-            events.append(CoverCreatedEvent(event.dest_path))
-        elif source_match_cover and dest_match_cover:
-            events.append(CoverMovedEvent(event.src_path, event.dest_path))
-
-    def _transform_file_event(self, event):
-        """Transform file events into other events."""
-        events = []
-        source_match_comic = self._match_comic_suffix(event.src_path)
-        if event.event_type == EVENT_TYPE_MOVED:
-            self._transform_file_move_event(event, events, source_match_comic)
-            self._transform_file_move_event_to_cover_event(event, events)
-        elif source_match_comic:
-            events.append(event)
-        else:
-            source_match_cover = self._match_folder_cover(event.src_path)
-            if source_match_cover and (
-                event_class := COVERS_EVENT_TYPE_MAP.get(event.event_type)
-            ):
-                # Convert to cover type
-                event = event_class(event.src_path)
-                events.append(event)
-        return events
-
-    def dispatch(self, event):
-        """Send only valid codex events to the EventBatcher."""
-        try:
-            if event.event_type in self.IGNORED_EVENTS or (
-                event.is_directory and event.event_type == EVENT_TYPE_CREATED
-            ):
-                # Directories are only created by comics
-                return
-
-            events = self._transform_file_event(event)
-
-            # Send it to the EventBatcher
-            for sub_event in events:
-                task = WatchdogEventTask(self.library_pk, sub_event)
-                self.librarian_queue.put(task)
-
-            # Do not call super dispatch to call stub event dispatchers
-        except Exception:
-            self.log.exception(f"{self.__class__.__name__} dispatch")
-
-
-class CodexCustomCoverEventHandler(CodexEventHandlerBase):
-    """Special event handler for the custom cover dir."""
-
-    @classmethod
-    def _match_group_cover_image(cls, path_str: str) -> bool:
-        path = Path(path_str)
-        parent = path.parent
-        if (
-            parent.parent != CUSTOM_COVERS_DIR
-            or str(parent.name) not in _GROUP_COVERS_DIRS
-        ):
-            return False
-        return cls._match_image_suffix(path)
-
-    @classmethod
-    def _transform_event_moved(cls, event, src_cover_match):
-        """Get a cover event for a file moved event."""
-        send_event = None
-        dest_cover_match = cls._match_group_cover_image(event.dest_path)
-        if src_cover_match and dest_cover_match:
-            send_event = CoverMovedEvent(event.src_path, event.dest_path)
-        elif not src_cover_match and dest_cover_match:
-            send_event = CoverCreatedEvent(event.dest_path)
-        elif src_cover_match and not dest_cover_match:
-            send_event = CoverDeletedEvent(event.src_path)
-        return send_event
-
-    def dispatch(self, event):
-        """Send only valid cover events to the EventBatcher."""
-        send_event = None
-        try:
-            if event.is_directory or event.event_type in self.IGNORED_EVENTS:
-                return
-            src_cover_match = self._match_group_cover_image(str(event.src_path))
-            if event.event_type == EVENT_TYPE_MOVED:
-                send_event = self._transform_event_moved(event, src_cover_match)
-            elif src_cover_match:
-                event_class = COVERS_EVENT_TYPE_MAP.get(event.event_type)
-                if not event_class:
-                    return
-                send_event = event_class(event.src_path)
-
-            if send_event:
-                task = WatchdogEventTask(self.library_pk, send_event)
-                self.librarian_queue.put(task)
-        except Exception:
-            self.log.exception(f"{self.__class__.__name__} dispatch")
+    event_type = EVENT_TYPE_FINISH_POLL

codex/librarian/watchdog/handlers.py

@@ -0,0 +1,203 @@
+"""Watchdog Event Handlers."""
+
+from abc import ABC, abstractmethod
+from multiprocessing import Queue
+from os import fsdecode
+from pathlib import Path
+
+from typing_extensions import override
+from watchdog.events import (
+    EVENT_TYPE_MOVED,
+    FileCreatedEvent,
+    FileDeletedEvent,
+    FileSystemEvent,
+    FileSystemEventHandler,
+)
+
+from codex.librarian.watchdog.const import COMIC_MATCHER, COVERS_EVENT_TYPE_MAP
+from codex.librarian.watchdog.events import (
+    CodexPollEvent,
+    CoverCreatedEvent,
+    CoverDeletedEvent,
+    CoverMovedEvent,
+)
+from codex.librarian.watchdog.tasks import WatchdogEventTask
+from codex.librarian.worker import WorkerMixin
+from codex.models import CustomCover
+from codex.settings import CUSTOM_COVERS_DIR, CUSTOM_COVERS_GROUP_DIRS
+
+_IMAGE_EXTS = frozenset({"jpg", "jpeg", "webp", "png", "gif", "bmp"})
+_GROUP_COVERS_DIRS = frozenset(CUSTOM_COVERS_GROUP_DIRS)
+
+
+class CodexEventHandlerBase(WorkerMixin, FileSystemEventHandler, ABC):
+    """Base class for Codex Event Handlers."""
+
+    def __init__(
+        self,
+        library_pk: int,
+        *args,
+        logger_,
+        librarian_queue: Queue,
+        db_write_lock,
+        **kwargs,
+    ):
+        """Let us send along he library id."""
+        self.library_pk = library_pk
+        self.init_worker(logger_, librarian_queue, db_write_lock)
+        super().__init__(*args, **kwargs)
+
+    @staticmethod
+    def _match_image_suffix(path: Path) -> bool:
+        return path and len(path.suffix) > 1 and path.suffix[1:] in _IMAGE_EXTS
+
+    @classmethod
+    @abstractmethod
+    def transform_file_event(
+        cls, event: FileSystemEvent
+    ) -> tuple[FileSystemEvent, ...]:
+        """Transform events into valid cover events."""
+        raise NotImplementedError
+
+    @override
+    def dispatch(self, event):
+        """Send only valid codex events to the EventBatcher."""
+        try:
+            if isinstance(event, CodexPollEvent):
+                events = [event]
+            else:
+                events = self.transform_file_event(event)
+
+            # Send it to the EventBatcher
+            for sub_event in events:
+                task = WatchdogEventTask(self.library_pk, sub_event)
+                self.librarian_queue.put(task)
+
+            # Do not call super dispatch to call stub event dispatchers
+        except Exception:
+            self.log.exception(f"{self.__class__.__name__} dispatch")
+
+
+class CodexLibraryEventHandler(CodexEventHandlerBase):
+    """Handle watchdog events for comics in a library."""
+
+    @staticmethod
+    def _match_comic_suffix(path):
+        """Match a supported comic suffix."""
+        if not path:
+            return False
+        # We don't care about general suffixes. Only length four.
+        suffix = path[-4:]
+        suffix = fsdecode(suffix)
+        return COMIC_MATCHER.match(suffix) is not None
+
+    @classmethod
+    def _match_folder_cover(cls, path_str: str | bytes) -> bool:
+        if not path_str:
+            return False
+        if isinstance(path_str, bytes):
+            path_str = path_str.decode()
+        path = Path(path_str)
+        return path.stem == CustomCover.FOLDER_COVER_STEM and cls._match_image_suffix(
+            path
+        )
+
+    @classmethod
+    def _transform_file_move_event(
+        cls,
+        event,
+        events,
+        source_match_comic,
+    ):
+        """Create file events for file moves."""
+        dest_match_comic = cls._match_comic_suffix(event.dest_path)
+        if not source_match_comic and dest_match_comic:
+            # Moved from an ignored file extension into a comic type,
+            # so create a new comic.
+            events.append(FileCreatedEvent(event.dest_path))
+        elif source_match_comic and not dest_match_comic:
+            # moved into something that's not a comic name so delete
+            events.append(FileDeletedEvent(event.src_path))
+        elif source_match_comic and dest_match_comic:
+            events.append(event)
+
+    @classmethod
+    def _transform_file_move_event_to_cover_event(cls, event, events):
+        """Create cover events for file moves."""
+        source_match_cover = cls._match_folder_cover(event.src_path)
+        dest_match_cover = cls._match_folder_cover(event.dest_path)
+        if source_match_cover and not dest_match_cover:
+            events.append(CoverDeletedEvent(event.src_path))
+        elif not source_match_cover and dest_match_cover:
+            events.append(CoverCreatedEvent(event.dest_path))
+        elif source_match_cover and dest_match_cover:
+            events.append(CoverMovedEvent(event.src_path, event.dest_path))
+
+    @override
+    @classmethod
+    def transform_file_event(cls, event: FileSystemEvent) -> tuple[FileSystemEvent]:
+        """Transform file events into other events."""
+        events = []
+        source_match_comic = cls._match_comic_suffix(event.src_path)
+        if event.event_type == EVENT_TYPE_MOVED:
+            cls._transform_file_move_event(event, events, source_match_comic)
+            cls._transform_file_move_event_to_cover_event(event, events)
+        elif source_match_comic:
+            events.append(event)
+        else:
+            source_match_cover = cls._match_folder_cover(event.src_path)
+            if source_match_cover and (
+                event_class := COVERS_EVENT_TYPE_MAP.get(event.event_type)
+            ):
+                # Convert to cover type
+                event = event_class(event.src_path)
+                events.append(event)
+        return tuple(events)
+
+
+class CodexCustomCoverEventHandler(CodexEventHandlerBase):
+    """Special event handler for the custom cover dir."""
+
+    @classmethod
+    def _match_group_cover_image(cls, path_str: str) -> bool:
+        path = Path(path_str)
+        parent = path.parent
+        if (
+            parent.parent != CUSTOM_COVERS_DIR
+            or str(parent.name) not in _GROUP_COVERS_DIRS
+        ):
+            return False
+        return cls._match_image_suffix(path)
+
+    @classmethod
+    def _transform_event_moved(cls, event, src_cover_match):
+        """Get a cover event for a file moved event."""
+        send_event = None
+        dest_cover_match = cls._match_group_cover_image(event.dest_path)
+        if src_cover_match and dest_cover_match:
+            send_event = CoverMovedEvent(event.src_path, event.dest_path)
+        elif not src_cover_match and dest_cover_match:
+            send_event = CoverCreatedEvent(event.dest_path)
+        elif src_cover_match and not dest_cover_match:
+            send_event = CoverDeletedEvent(event.src_path)
+        return send_event
+
+    @override
+    @classmethod
+    def transform_file_event(
+        cls, event: FileSystemEvent
+    ) -> tuple[FileSystemEvent, ...]:
+        """Transform events into valid cover events."""
+        send_events = []
+        if event.is_directory:
+            return tuple(send_events)
+        src_cover_match = cls._match_group_cover_image(str(event.src_path))
+        if event.event_type == EVENT_TYPE_MOVED:
+            if send_event := cls._transform_event_moved(event, src_cover_match):
+                send_events.append(send_event)
+        elif src_cover_match and (
+            event_class := COVERS_EVENT_TYPE_MAP.get(event.event_type)
+        ):
+            send_event = event_class(event.src_path)
+            send_events.append(send_event)
+        return tuple(send_events)

codex/librarian/watchdog/memory.py

@@ -0,0 +1,56 @@
+"""Detect how much memory we're working with."""
+
+import resource
+from contextlib import suppress
+from pathlib import Path
+
+from psutil import virtual_memory
+
+# cgroups1
+MEMORY_STAT_PATH = "/sys/fs/cgroup/memory/memory.stat"
+
+# cgroups2
+MEMORY_MAX_PATH = "/sys/fs/cgroup/memory.max"
+
+DIVISORS = {"b": 1, "k": 1024, "m": 1024**2, "g": 1024**3}
+
+
+def _get_cgroups2_mem_limit():
+    """Get mem limit from cgroups2."""
+    with Path(MEMORY_MAX_PATH).open("r") as limit:
+        return int(limit.read())
+
+
+def _get_cgroups1_mem_limit():
+    """Get mem limit from cgroups1."""
+    with Path(MEMORY_STAT_PATH).open("r") as mem_stat_file:
+        for line in mem_stat_file:
+            parts = line.split()
+            if not parts or len(parts) < 2:  # noqa: PLR2004
+                continue
+            if "hierarchical_memory_limit" in parts[0]:
+                return int(parts[1])
+    return None
+
+
+def get_mem_limit(divisor="b"):
+    """
+    Get the current memlimit.
+
+    If we're in a container set the limit too.
+    """
+    mem_limit = None
+    api_funcs = (_get_cgroups2_mem_limit, _get_cgroups1_mem_limit)
+    for func in api_funcs:
+        with suppress(Exception):
+            mem_limit = func()
+        if mem_limit:
+            break
+    if mem_limit:
+        # Set the process memlimit.
+        resource.setrlimit(resource.RLIMIT_AS, (mem_limit, mem_limit))
+
+    if not mem_limit:
+        # get the raw host memory
+        mem_limit = virtual_memory().total
+    return mem_limit / DIVISORS.get(divisor, 1)