codex 1.7.14__py3-none-any.whl → 1.8.0__py3-none-any.whl

codex/librarian/status_controller.py

@@ -0,0 +1,196 @@
+"""Librarian Status."""
+
+from collections.abc import Iterable
+from inspect import isclass
+from multiprocessing import Queue
+from time import time
+from types import MappingProxyType
+from typing import Any
+
+from django.db.models.functions.datetime import Now
+from django.db.models.query import Q
+from django.utils.timezone import datetime, now, timedelta
+from loguru._logger import Logger
+
+from codex.librarian.notifier.tasks import LIBRARIAN_STATUS_TASK
+from codex.librarian.status import Status
+from codex.models import LibrarianStatus
+
+
+def get_default(field):
+    """Get the default value for the model field."""
+    return LibrarianStatus._meta.get_field(field).get_default()
+
+
+DEFAULT_FIELDS = ("preactive", "complete", "total", "active", "subtitle")
+STATUS_DEFAULTS = {field: get_default(field) for field in DEFAULT_FIELDS}
+
+
+class StatusController:
+    """Run operations on the LibrarianStatus table."""
+
+    _UPDATE_DELTA = 5
+
+    def __init__(self, logger_: Logger, librarian_queue: Queue):
+        """Iinitialize logger and librarian queue."""
+        self.log = logger_
+        self.librarian_queue = librarian_queue
+
+    def _enqueue_notifier_task(self, *, notify: bool = True):
+        """Notify the status has changed."""
+        if not notify:
+            return
+        self.librarian_queue.put(LIBRARIAN_STATUS_TASK)
+
+    def _loggit(self, level: str, status: Status):
+        """Log with a ? in place of none."""
+        msg = f"{status.title()} {status.subtitle}".strip()
+        msg += ": "
+        if status.complete is None and status.total is None:
+            msg += "In progress"
+        else:
+            count = "?" if status.complete is None else status.complete
+            total = "?" if status.total is None else status.total
+            msg += f"{count}/{total}"
+
+        self.log.log(level, msg)
+
+    def _update(
+        self,
+        status: Status,
+        *,
+        notify: bool,
+        active: datetime | None = None,
+        preactive: datetime | None = None,
+    ):
+        """Start a librarian status."""
+        try:
+            updates: dict[str, Any] = {
+                "complete": status.complete,
+                "total": status.total,
+                "updated_at": Now(),
+            }
+            if preactive is not None:
+                updates["preactive"] = preactive
+            if active:
+                updates["active"] = active
+            if status.subtitle:
+                updates["subtitle"] = status.subtitle
+            LibrarianStatus.objects.filter(status_type=status.CODE).update(**updates)
+            self._enqueue_notifier_task(notify=notify)
+            self._loggit("DEBUG", status)
+            status.since_updated = time()
+        except Exception:
+            self.log.exception(f"Update status: {status.title()}")
+
+    def start(
+        self,
+        status: Status,
+        *,
+        notify: bool = True,
+        preactive: datetime | None = None,
+    ):
+        """Start a librarian status."""
+        status.start()
+        self._update(status, notify=notify, preactive=preactive, active=now())
+
+    def start_many(self, statii: Iterable[Status | type[Status]]):
+        """Start many librarian statuses."""
+        for index, status_or_class in enumerate(statii):
+            status = status_or_class() if isclass(status_or_class) else status_or_class
+            status.start()
+            pad_ms = index * 100  # for order
+            preactive = now() + timedelta(milliseconds=pad_ms)
+            self._update(status, notify=False, preactive=preactive)
+        self._enqueue_notifier_task(notify=True)
+
+    def update(self, status: Status, *, notify: bool = True):
+        """Update a librarian status."""
+        if time() - status.since_updated < self._UPDATE_DELTA:
+            # noop unless time has expired.
+            return
+        self._update(status, notify=notify)
+
+    def _log_finish(self, status: Status):
+        """Log finish of status with stats."""
+        level = "INFO"
+        suffix = ""
+        if elapsed := status.elapsed():
+            suffix += f" in {elapsed}"
+        if status.SINGLE:
+            count = ""
+        elif count := status.complete:
+            count = str(count)
+            if persecond := status.per_second():
+                suffix += f" at a rate of {persecond}"
+        else:
+            count = "no"
+            level = "DEBUG"
+
+        if status.log_success:
+            level = "SUCCESS"
+
+        prefix_parts = [status.verbed()]
+        if count:
+            prefix_parts.append(count)
+        prefix_parts.append(status.ITEM_NAME)
+        prefix = " ".join(prefix_parts)
+
+        self.log.log(level, f"{prefix}{suffix}.")
+
+    @staticmethod
+    def _finish_status_prepare(
+        positive_statii: MappingProxyType[str, Status | type[Status]],
+        updates: dict[str, Any],
+    ):
+        # Filter on submitted codes or all statii if nothing.
+        ls_filter_dict: dict[str, Any] = (
+            {"status_type__in": positive_statii.keys()} if positive_statii else {}
+        )
+        # Filter all active or preactive statii
+        ls_filter = (Q(active__isnull=False) | Q(preactive__isnull=False)) & Q(
+            **ls_filter_dict
+        )
+        lses = LibrarianStatus.objects.filter(ls_filter)
+        update_ls = []
+        log_statii = []
+        for ls in lses.iterator():
+            for key, value in updates.items():
+                setattr(ls, key, value)
+            update_ls.append(ls)
+            status = positive_statii[ls.status_type]
+            if isinstance(status, Status):
+                log_statii.append(status)
+        return update_ls, log_statii
+
+    def finish_many(
+        self, statii: Iterable[Status | type[Status] | None], *, notify: bool = True
+    ):
+        """Finish all librarian statuses."""
+        positive_statii: MappingProxyType[str, Status | type[Status]] = (
+            MappingProxyType({status.CODE: status for status in statii if status})
+        )
+        try:
+            if statii and not positive_statii:
+                # if statii has elements but they were all None, this is a noop.
+                # But if statii was empty this is a finish all command.
+                return
+            updates = {**STATUS_DEFAULTS, "updated_at": Now()}
+            update_ls, log_statii = self._finish_status_prepare(
+                positive_statii, updates
+            )
+            LibrarianStatus.objects.bulk_update(update_ls, tuple(updates.keys()))
+            self._enqueue_notifier_task(notify=notify)
+            if not positive_statii:
+                self.log.info("Cleared all librarian statuses")
+            for status in log_statii:
+                self._log_finish(status)
+        except Exception as exc:
+            self.log.warning(f"Finish status {positive_statii}: {exc}")
+
+    def finish(self, status: Status | None, *, notify: bool = True):
+        """Finish a librarian status."""
+        try:
+            self.finish_many((status,), notify=notify)
+        except Exception as exc:
+            self.log.warning(exc)

codex/librarian/tasks.py

@@ -1,15 +1,5 @@
 """Librarian Tasks."""
 
-from dataclasses import dataclass, field
-
-
-@dataclass(order=True)
-class DelayedTasks:
-    """A list of tasks to start on a delay."""
-
-    until: float
-    tasks: tuple = field(compare=False)
-
 
 class LibrarianShutdownTask:
     """Signal task."""

codex/librarian/telemeter/scheduled_time.py

@@ -3,6 +3,9 @@
 from datetime import datetime, timedelta, timezone
 from uuid import UUID
 
+from loguru._logger import Logger
+
+from codex.choices.admin import AdminFlagChoices
 from codex.librarian.telemeter.telemeter import get_telemeter_timestamp
 from codex.models.admin import AdminFlag
 
@@ -45,17 +48,21 @@ def _get_scheduled_time(ts):
     return telemeter_time
 
 
-def get_telemeter_time(log):
+def get_telemeter_time(log: Logger):
     """Get the time to send telemetry."""
     # Should we schedule telemeter at all?
-    if not AdminFlag.objects.get(key=AdminFlag.FlagChoices.SEND_TELEMETRY.value).on:
-        log.debug("Telemeter disabled. Not scheduled.")
+    if (
+        not AdminFlag.objects.only("on")
+        .get(key=AdminFlagChoices.SEND_TELEMETRY.value)
+        .on
+    ):
+        log.trace("Telemeter disabled. Not scheduled.")
         return 0
 
     ts = get_telemeter_timestamp()
 
     if _is_created_recently(ts):
-        log.debug("Telemeter created recently. Not scheduled.")
+        log.trace("Telemeter created recently. Not scheduled.")
         return 0
 
     return _get_scheduled_time(ts)

codex/librarian/telemeter/stats.py

@@ -9,7 +9,6 @@ from caseconverter import snakecase
 from django.contrib.sessions.models import Session
 from django.db.models import Count
 
-from codex.logger.logger import get_logger
 from codex.models import (
     Comic,
     Library,
@@ -18,7 +17,6 @@ from codex.version import VERSION
 from codex.views.const import CONFIG_MODELS, METADATA_MODELS, STATS_GROUP_MODELS
 from codex.views.session import SessionView
 
-LOG = get_logger(__name__)
 _KEY_MODELS_MAP = MappingProxyType(
     {
         "config": CONFIG_MODELS,

codex/librarian/telemeter/tasks.py

@@ -2,7 +2,9 @@
 
 from dataclasses import dataclass
 
+from codex.librarian.bookmark.tasks import BookmarkTask
+
 
 @dataclass
-class TelemeterTask:
+class TelemeterTask(BookmarkTask):
     """Send telemetry."""

codex/librarian/telemeter/telemeter.py

@@ -2,12 +2,12 @@
 
 import json
 from base64 import a85decode
-from logging import getLogger
 from lzma import compress
 from uuid import uuid4
 
 from requests import Session
 
+from codex.choices.admin import AdminFlagChoices
 from codex.librarian.telemeter.stats import CodexStats
 from codex.models.admin import AdminFlag, Timestamp
 
@@ -28,12 +28,10 @@ _HEADERS = {"Content-Type": "application/xz"}
 _POST = _BASE + f"/stats/{_APP_NAME}/{_VERSION}"
 _TIMEOUT = 5
 
-LOG = getLogger(__name__)
-
 
 def get_telemeter_timestamp():
     """Get or create timestamp."""
-    key = Timestamp.TimestampChoices.TELEMETER_SENT.value
+    key = Timestamp.Choices.TELEMETER_SENT.value
     defaults = {"key": key}
     ts, _ = Timestamp.objects.get_or_create(defaults=defaults, key=key)
     if not ts.version:
@@ -54,21 +52,29 @@ def _post_stats(data):
         response.raise_for_status()
 
 
-def _send_telemetry():
+def _send_telemetry(uuid):
     """Send telemetry to server."""
-    if not AdminFlag.objects.get(key=AdminFlag.FlagChoices.SEND_TELEMETRY.value).on:
+    if (
+        not AdminFlag.objects.only("on")
+        .get(key=AdminFlagChoices.SEND_TELEMETRY.value)
+        .on
+    ):
         reason = "Send Telemetry flag is off."
         raise ValueError(reason)
-    ts = get_telemeter_timestamp()
     stats = CodexStats().get()
-    data = {"stats": stats, "uuid": ts.version}
+    data = {"stats": stats, "uuid": uuid}
     _post_stats(data)
-    ts.save()  # update updated_at
 
 
 def send_telemetry(log):
     """Send anonymous telemetry during one window per week."""
     try:
-        _send_telemetry()
+        ts = get_telemeter_timestamp()
+        try:
+            _send_telemetry(ts.version)
+        except Exception as exc:
+            log.debug(f"Failed to send anonyomous stats: {exc}")
+        # update updated_at, even on failure to prevent rapid rescheudling.
+        ts.save()
     except Exception as exc:
-        log.debug(f"Failed to send anonyomous stats: {exc}")
+        log.debug(f"Failed to get or set telemeter timestamp: {exc}")

codex/librarian/threads.py

@@ -0,0 +1,148 @@
+"""Abstract Thread worker for doing queued tasks."""
+
+import time
+from abc import ABC, abstractmethod
+from multiprocessing.queues import Queue
+from queue import Empty, SimpleQueue
+from threading import Thread
+
+from loguru._logger import Logger
+from typing_extensions import override
+
+from codex.librarian.worker import WorkerStatusMixin
+
+
+class BreakLoopError(Exception):
+    """Simple way to break out of function nested loop."""
+
+
+class NamedThread(Thread, WorkerStatusMixin, ABC):
+    """A thread that sets its name for ps."""
+
+    def __init__(
+        self,
+        logger_: Logger,
+        librarian_queue: Queue,
+        db_write_lock,
+        *args,
+        **kwargs,
+    ):
+        """Initialize queues."""
+        self.init_worker(logger_, librarian_queue, db_write_lock)
+        super().__init__(*args, name=self.__class__.__name__, **kwargs)
+
+    def run_start(self):
+        """First thing to do when running a new thread."""
+        self.log.debug(f"Started {self.name}")
+
+
+class QueuedThread(NamedThread, ABC):
+    """Abstract Thread worker for doing queued tasks."""
+
+    SHUTDOWN_MSG = "shutdown"
+    SHUTDOWN_TIMEOUT = 5
+
+    def __init__(self, *args, **kwargs):
+        """Initialize with overridden name and as a daemon thread."""
+        self.queue = kwargs.pop("queue", SimpleQueue())
+        super().__init__(*args, daemon=True, **kwargs)
+
+    @abstractmethod
+    def process_item(self, item):
+        """Process one item from the queue."""
+        raise NotImplementedError
+
+    def get_timeout(self) -> float | None:
+        """Set no timeout by default."""
+        return
+
+    def timed_out(self):
+        """Override to things on queue timeout."""
+
+    def _check_item(self):
+        """Get items, with timeout. Check for shutdown and Empty."""
+        timeout = self.get_timeout()
+        try:
+            item = self.queue.get(timeout=timeout)
+            if item == self.SHUTDOWN_MSG:
+                raise BreakLoopError
+            self.process_item(item)
+        except Empty:
+            self.timed_out()
+
+    @override
+    def run(self):
+        """Run thread loop."""
+        self.run_start()
+        while True:
+            try:
+                self._check_item()
+            except BreakLoopError:
+                break
+            except Exception:
+                self.log.exception(f"{self.__class__.__name__} crashed:")
+        self.log.debug(f"Stopped {self.__class__.__name__}")
+
+    def stop(self):
+        """Stop the thread."""
+        self.queue.put(self.SHUTDOWN_MSG)
+
+    @override
+    def join(self, timeout=None):
+        """End the thread."""
+        self.log.debug(f"Waiting for {self.__class__.__name__} to join.")
+        super().join(self.SHUTDOWN_TIMEOUT)
+        self.log.debug(f"{self.__class__.__name__} joined.")
+
+
+class AggregateMessageQueuedThread(QueuedThread, ABC):
+    """Abstract Thread worker for buffering and aggregating messages."""
+
+    FLOOD_DELAY = 1.0
+    MAX_DELAY = 5.0
+
+    def __init__(self, *args, **kwargs):
+        """Initialize the cache."""
+        self.cache = {}
+        self._last_send = time.time()
+        super().__init__(*args, **kwargs)
+
+    def set_last_send(self):
+        """Set the last send time to now."""
+        self._last_send = time.time()
+
+    @override
+    def get_timeout(self):
+        """Aggregate queue has a conditional timeout."""
+        return self.FLOOD_DELAY if self.cache else None
+
+    @abstractmethod
+    def aggregate_items(self, item):
+        """Abstract method for aggregating items."""
+        raise NotImplementedError
+
+    @abstractmethod
+    def send_all_items(self):
+        """Abstract method for sending all items."""
+        raise NotImplementedError
+
+    def cleanup_cache(self, keys):
+        """Remove sent messages from the cache and record send times."""
+        for key in keys:
+            self.cache.pop(key, None)
+        self.set_last_send()
+
+    @override
+    def process_item(self, item):
+        """Aggregate items and sleep in case there are more."""
+        self.aggregate_items(item)
+        since_last_timed_out = time.time() - self._last_send
+        waited_too_long = since_last_timed_out > self.MAX_DELAY
+        if waited_too_long:
+            self.timed_out()
+
+    @override
+    def timed_out(self):
+        """Send the items and set when we did this."""
+        self.send_all_items()
+        self.set_last_send()

codex/librarian/watchdog/const.py

@@ -0,0 +1,139 @@
+"""Watchdog consts."""
+
+import re
+from types import MappingProxyType
+
+from comicbox.box import Comicbox
+from loguru import logger
+from watchdog.events import (
+    EVENT_TYPE_CREATED,
+    EVENT_TYPE_DELETED,
+    EVENT_TYPE_MODIFIED,
+    DirDeletedEvent,
+    DirModifiedEvent,
+    DirMovedEvent,
+    FileCreatedEvent,
+    FileDeletedEvent,
+    FileModifiedEvent,
+    FileMovedEvent,
+    FileSystemEvent,
+)
+
+from codex.librarian.watchdog.events import (
+    CodexPollEvent,
+    CoverCreatedEvent,
+    CoverDeletedEvent,
+    CoverModifiedEvent,
+    CoverMovedEvent,
+)
+
+#############
+# Observers #
+#############
+EVENT_FILTER: tuple[type[FileSystemEvent], ...] = (
+    # FileClosed,
+    # FileClosedNoWriteEvent,
+    FileCreatedEvent,
+    FileDeletedEvent,
+    FileModifiedEvent,
+    FileMovedEvent,
+    # FileSystemMovedEvent
+    # FileOpenedEvent,
+    # DirCreatedEvent,
+    DirDeletedEvent,
+    DirMovedEvent,
+    DirModifiedEvent,
+)
+
+###########
+# Emitter #
+###########
+ATTR_EVENT_MAP = MappingProxyType(
+    {
+        "files_deleted": FileDeletedEvent,
+        "files_modified": FileModifiedEvent,
+        "files_created": FileCreatedEvent,
+        "files_moved": FileMovedEvent,
+        "dirs_deleted": DirDeletedEvent,
+        "dirs_modified": DirModifiedEvent,
+        "dirs_moved": DirMovedEvent,
+    }
+)
+DIR_NOT_FOUND_TIMEOUT = 15 * 60
+POLLING_EVENT_FILTER: tuple[type[FileSystemEvent], ...] = (
+    *EVENT_FILTER,
+    CodexPollEvent,
+)
+DOCKER_UNMOUNTED_FN = "DOCKER_UNMOUNTED_VOLUME"
+
+############
+# Handlers #
+############
+
+
+def _get_comic_matcher():
+    comic_regex = r"\.(cb[zt7"
+    unsupported = []
+    if Comicbox.is_unrar_supported():
+        comic_regex += r"r"
+    else:
+        unsupported.append("CBR")
+    comic_regex += r"]"
+
+    if Comicbox.is_pdf_supported():
+        comic_regex += r"|pdf"
+    else:
+        unsupported.append("PDF")
+    comic_regex += ")$"
+    if unsupported:
+        un_str = ", ".join(unsupported)
+        logger.warning(f"Cannot detect or read from {un_str} archives")
+    return re.compile(comic_regex, re.IGNORECASE)
+
+
+COMIC_MATCHER = _get_comic_matcher()
+COVERS_EVENT_TYPE_MAP = MappingProxyType(
+    {
+        EVENT_TYPE_MODIFIED: CoverModifiedEvent,
+        EVENT_TYPE_CREATED: CoverCreatedEvent,
+        EVENT_TYPE_DELETED: CoverDeletedEvent,
+    }
+)
+
+
+####################
+# Event Aggregator #
+####################
+EVENT_CLASS_DIFF_ATTR_MAP = MappingProxyType(
+    {
+        FileDeletedEvent: "files_deleted",
+        FileModifiedEvent: "files_modified",
+        FileCreatedEvent: "files_created",
+        DirDeletedEvent: "dirs_deleted",
+        DirModifiedEvent: "dirs_modified",
+    }
+)
+EVENT_MOVED_CLASS_DIFF_ATTR_MAP = MappingProxyType(
+    {FileMovedEvent: "files_moved", DirMovedEvent: "dirs_moved"}
+)
+EVENT_COVERS_DIFF_ATTR_MAP = MappingProxyType(
+    {
+        CoverCreatedEvent: "covers_created",
+        CoverDeletedEvent: "covers_deleted",
+        CoverModifiedEvent: "covers_modified",
+    }
+)
+EVENT_COVERS_MOVED_CLASS_DIFF_ATTR_MAP = MappingProxyType(
+    {CoverMovedEvent: "covers_moved"}
+)
+
+EVENT_CLASS_DIFF_ALL_MAP: MappingProxyType[type[FileSystemEvent], str] = (
+    MappingProxyType(
+        {
+            **EVENT_CLASS_DIFF_ATTR_MAP,
+            **EVENT_MOVED_CLASS_DIFF_ATTR_MAP,
+            **EVENT_COVERS_DIFF_ATTR_MAP,
+            **EVENT_COVERS_MOVED_CLASS_DIFF_ATTR_MAP,
+        }
+    )
+)

codex/librarian/watchdog/db_snapshot.py

@@ -6,11 +6,10 @@ from pathlib import Path
 
 from watchdog.utils.dirsnapshot import DirectorySnapshot
 
-from codex.logger_base import LoggerBaseMixin
 from codex.models import Comic, CustomCover, FailedImport, Folder
 
 
-class CodexDatabaseSnapshot(DirectorySnapshot, LoggerBaseMixin):
+class CodexDatabaseSnapshot(DirectorySnapshot):
     """Take snapshots from the Codex database."""
 
     MODELS = (Folder, Comic, FailedImport, CustomCover)
@@ -58,19 +57,26 @@ class CodexDatabaseSnapshot(DirectorySnapshot, LoggerBaseMixin):
         i = (st.st_ino, st.st_dev)
         self._inode_to_path[i] = path
 
-    def __init__(
+    def __init__(  # noqa: PLR0913 # pyright: ignore[reportMissingSuperCall]
         self,
         path,
-        _recursive=True,  # noqa: FBT002 unused, always recursive
+        *,
+        recursive=True,  # noqa: ARG002
         stat=os.stat,
-        _listdir=os.listdir,  # unused for database
-        force=False,  # noqa: FBT002
-        log_queue=None,
-        covers_only=False,  # noqa: FBT002
+        listdir=os.scandir,  # noqa: ARG002 unused for database
+        logger_=None,
+        force=False,
+        covers_only=False,
     ):
         """Initialize like DirectorySnapshot but use a database walk."""
+        # Do not call super().__init__(), because it walks.
+        if not logger_:
+            reason = "logger_ must be set"
+            raise ValueError(reason)
+
+        self.log = logger_
+        self.stat = stat
         self._covers_only = covers_only
-        self.init_logger(log_queue)
         self._stat_info = {}
         self._inode_to_path = {}
         if not Path(path).is_dir():
@@ -78,7 +84,7 @@ class CodexDatabaseSnapshot(DirectorySnapshot, LoggerBaseMixin):
             return
 
         # Add the library root
-        root_stat = stat(path)
+        root_stat = self.stat(path)
         self._set_lookups(path, root_stat)
 
         for wp in chain.from_iterable(self._walk(path)):