codex 1.6.8__py3-none-any.whl → 1.6.9a0__py3-none-any.whl

codex/librarian/janitor/tasks.py

@@ -8,6 +8,20 @@ class JanitorTask:
     """Tasks for the janitor."""
 
 
+@dataclass
+class JanitorLatestVersionTask(JanitorTask):
+    """Get the latest version."""
+
+    force: bool = False
+
+
+@dataclass
+class JanitorUpdateTask(JanitorTask):
+    """Task for updater."""
+
+    force: bool = False
+
+
 @dataclass
 class JanitorBackupTask(JanitorTask):
     """Backup the database."""
@@ -23,13 +37,6 @@ class JanitorShutdownTask(JanitorTask):
     """for shutdown."""
 
 
-@dataclass
-class JanitorUpdateTask(JanitorTask):
-    """Task for updater."""
-
-    force: bool
-
-
 @dataclass
 class JanitorVacuumTask(JanitorTask):
     """Vacuum the database."""

codex/librarian/janitor/time.py

@@ -0,0 +1,12 @@
+"""Janitor Scheduled time."""
+
+from datetime import datetime, time, timedelta
+
+from django.utils import timezone as django_timezone
+
+
+def get_janitor_time(_log):
+    """Get midnight relative to now."""
+    tomorrow = django_timezone.now() + timedelta(days=1)
+    tomorrow = tomorrow.astimezone()
+    return datetime.combine(tomorrow, time.min).astimezone()

codex/librarian/janitor/update.py

@@ -5,17 +5,36 @@ import signal
 import subprocess
 import sys
 
+from versio.version import Version
+
 from codex.librarian.janitor.status import JanitorStatusTypes
 from codex.librarian.tasks import LibrarianShutdownTask
 from codex.models import AdminFlag
+from codex.models.admin import Timestamp
 from codex.status import Status
-from codex.version import PACKAGE_NAME, VERSION, get_version, is_outdated
+from codex.version import VERSION, get_version
 from codex.worker_base import WorkerBaseMixin
 
 
 class UpdateMixin(WorkerBaseMixin):
     """Update codex methods for janitor."""
 
+    def _is_outdated(self):
+        """Is codex outdated."""
+        result = False
+        ts = Timestamp.objects.get(key=Timestamp.TimestampChoices.CODEX_VERSION.value)
+        latest_version = ts.version
+        versio_latest_version = Version(latest_version)
+
+        installed_versio_version = Version(VERSION)
+        if versio_latest_version.parts[1] and not installed_versio_version.parts[1]:  # type: ignore
+            pre_blurb = "latest version is a prerelease. But installed version is not."
+        else:
+            result = versio_latest_version > installed_versio_version
+            pre_blurb = ""
+        self.log.info(f"{latest_version=} > {VERSION=} = {result}{pre_blurb}")
+        return result
+
     def update_codex(self, force=False):
         """Update the package and restart everything if the version changed."""
         status = Status(JanitorStatusTypes.CODEX_UPDATE)
@@ -27,7 +46,7 @@ class UpdateMixin(WorkerBaseMixin):
                 eau = AdminFlag.objects.only("on").get(
                     key=AdminFlag.FlagChoices.AUTO_UPDATE.value
                 )
-                if not eau.on or not is_outdated(PACKAGE_NAME):
+                if not eau.on or not self._is_outdated():
                     self.log.info("Codex is up to date.")
                     return
 

codex/librarian/janitor/vacuum.py

@@ -21,8 +21,7 @@ class VacuumMixin(WorkerBaseMixin):
             self.status_controller.start(status)
             old_size = DB_PATH.stat().st_size
             with connection.cursor() as cursor:
-                cursor.execute("PRAGMA analysis_limit=400;")
-                cursor.execute("PRAGMA optimize;")
+                cursor.execute("PRAGMA optimize")
                 cursor.execute("VACUUM")
                 cursor.execute("PRAGMA wal_checkpoint(TRUNCATE)")
             new_size = DB_PATH.stat().st_size

codex/librarian/librariand.py

@@ -9,6 +9,7 @@ from caseconverter import snakecase
 
 from codex.librarian.covers.coverd import CoverThread
 from codex.librarian.covers.tasks import CoverTask
+from codex.librarian.cron.crond import CronThread
 from codex.librarian.delayed_taskd import DelayedTasksThread
 from codex.librarian.importer.importerd import ComicImporterThread
 from codex.librarian.importer.tasks import (
@@ -16,7 +17,6 @@ from codex.librarian.importer.tasks import (
     ImportTask,
 )
 from codex.librarian.janitor.janitor import Janitor
-from codex.librarian.janitor.janitord import JanitorThread
 from codex.librarian.janitor.tasks import JanitorTask
 from codex.librarian.notifier.notifierd import NotifierThread
 from codex.librarian.notifier.tasks import NotifierTask
@@ -26,7 +26,9 @@ from codex.librarian.search.tasks import (
     SearchIndexerTask,
     SearchIndexRebuildIfDBChangedTask,
 )
-from codex.librarian.tasks import DelayedTasks, LibrarianShutdownTask
+from codex.librarian.tasks import DelayedTasks, LibrarianShutdownTask, WakeCronTask
+from codex.librarian.telemeter.tasks import TelemeterTask
+from codex.librarian.telemeter.telemeter import send_telemetry
 from codex.librarian.watchdog.event_batcherd import WatchdogEventBatcherThread
 from codex.librarian.watchdog.observers import (
     LibraryEventObserver,
@@ -52,7 +54,7 @@ class LibrarianDaemon(Process, LoggerBaseMixin):
         WatchdogEventBatcherThread,
         LibraryEventObserver,
         LibraryPollingObserver,
-        JanitorThread,
+        CronThread,
     )
     _THREAD_CLASS_MAP = MappingProxyType(
         {
@@ -105,6 +107,10 @@ class LibrarianDaemon(Process, LoggerBaseMixin):
                     self.log.debug("Told search indexers to stop for db updates.")
                 else:
                     self._threads.search_indexer_thread.queue.put(task)
+            case WakeCronTask():
+                self._threads.cron_thread.end_timeout()
+            case TelemeterTask():
+                send_telemetry(self.log)
             case JanitorTask():
                 self.janitor.run(task)
             case DelayedTasks():

codex/librarian/tasks.py

@@ -12,4 +12,8 @@ class DelayedTasks:
 
 
 class LibrarianShutdownTask:
-    """Empty task."""
+    """Signal task."""
+
+
+class WakeCronTask:
+    """Signal task."""

codex/librarian/telemeter/__init__.py

@@ -0,0 +1 @@
+"""Telemeter."""

codex/librarian/{stats.py → telemeter/stats.py}

@@ -111,8 +111,10 @@ class CodexStats:
 
         return user_stats, anon_session_count
 
-    def _get_platform(self, obj):
+    def _add_platform(self, obj):
         """Add dict of platform information to object."""
+        if self.params and "platform" not in self.params:
+            return
         platform = {
             "docker": self._is_docker(),
             "machine": machine(),
@@ -126,8 +128,10 @@ class CodexStats:
         }
         obj["platform"] = platform
 
-    def _get_config(self, obj):
+    def _add_config(self, obj):
         """Add dict of config informaation to object."""
+        if self.params and "config" not in self.params:
+            return
         config = self._get_model_counts("config")
         sessions, config["user_anonymous_count"] = self._get_session_stats()
         config["user_registered_count"] = config.pop("users_count", 0)
@@ -135,14 +139,18 @@ class CodexStats:
         obj["config"] = config
         obj["sessions"] = sessions
 
-    def _get_groups(self, obj):
+    def _add_groups(self, obj):
         """Add dict of groups information to object."""
+        if self.params and "groups" not in self.params:
+            return
         groups = self._get_model_counts("groups")
+        groups["issue_count"] = groups.pop("comic_count", 0)
         obj["groups"] = groups
 
-    @staticmethod
-    def _get_file_types(obj):
+    def _add_file_types(self, obj):
         """Query for file types."""
+        if self.params and "fileTypes" not in self.params:
+            return
         file_types = {}
         qs = (
             Comic.objects.values("file_type")
@@ -156,22 +164,19 @@ class CodexStats:
         sorted_fts = dict(sorted(file_types.items()))
         obj["file_types"] = sorted_fts
 
-    def _get_metadata(self, obj):
+    def _add_metadata(self, obj):
         """Add dict of metadata counts to object."""
+        if self.params and "metadata" not in self.params:
+            return
         metadata = self._get_model_counts("metadata")
         obj["metadata"] = metadata
 
     def get(self):
         """Construct the stats object."""
         obj = {}
-        if not self.params or "platform" in self.params:
-            self._get_platform(obj)
-        if not self.params or "config" in self.params:
-            self._get_config(obj)
-        if not self.params or "groups" in self.params:
-            self._get_groups(obj)
-        if not self.params or "fileTypes" in self.params:
-            self._get_file_types(obj)
-        if not self.params or "metadata" in self.params:
-            self._get_metadata(obj)
+        self._add_platform(obj)
+        self._add_config(obj)
+        self._add_groups(obj)
+        self._add_file_types(obj)
+        self._add_metadata(obj)
         return obj

codex/librarian/telemeter/tasks.py

@@ -0,0 +1,8 @@
+"""Telemter tasks."""
+
+from dataclasses import dataclass
+
+
+@dataclass
+class TelemeterTask:
+    """Send telemetry."""

codex/librarian/telemeter/telemeter.py

@@ -0,0 +1,80 @@
+"""Telemeter job."""
+
+import json
+from base64 import a85decode
+from logging import getLogger
+from lzma import compress
+from uuid import uuid4
+
+from requests import Session
+
+from codex.librarian.telemeter.stats import CodexStats
+from codex.models.admin import AdminFlag, Timestamp
+from codex.settings.settings import DEBUG
+
+# Version
+_APP_NAME = "codex"
+_VERSION = "1"
+
+# Sending
+# this isn't meant to fool you. it's meant to discourage lazy scraper bots.
+_BASE = "".join(
+    (
+        a85decode(b"BQS?8F#ks-@:XCm@;\\+").decode(),
+        a85decode(b"Ea`frF)to6Bk]hRFCB94/c").decode(),
+        a85decode(b"@rGmhGV*rI@:Wqi/n&^<").decode(),
+    )
+)
+_HEADERS = {"Content-Type": "application/xz"}
+_POST = _BASE + f"/stats/{_APP_NAME}/{_VERSION}"
+_TIMEOUT = 5
+
+LOG = getLogger(__name__)
+
+
+def get_telemeter_timestamp():
+    """Get or create timestamp."""
+    key = Timestamp.TimestampChoices.TELEMETER_SENT.value
+    defaults = {"key": key}
+    ts, _ = Timestamp.objects.get_or_create(defaults=defaults, key=key)
+    if not ts.version:
+        ts.version = str(uuid4())
+        ts.save()
+    return ts
+
+
+def _post_stats(log, data):
+    """Post telemetry to endpoint."""
+    data_json = json.dumps(data)
+    json_bytes = data_json.encode()
+    compressed_data = compress(json_bytes)
+    if DEBUG:
+        msg = f"{_POST} {len(compressed_data)}, {_HEADERS}, {_TIMEOUT}"
+        log.debug(msg)
+    else:
+        with Session() as session:
+            response = session.post(
+                _POST, data=compressed_data, headers=_HEADERS, timeout=_TIMEOUT
+            )
+            response.raise_for_status()
+
+
+def _send_telemetry(log):
+    """Send telemetry to server."""
+    if not AdminFlag.objects.get(key=AdminFlag.FlagChoices.SEND_TELEMETRY.value).on:
+        reason = "Send Telemetry flag is off."
+        raise ValueError(reason)
+    ts = get_telemeter_timestamp()
+    stats = CodexStats().get()
+    data = {"stats": stats, "uuid": ts.version}
+    _post_stats(log, data)
+    # log.debug("Sent anonymous stats")
+    ts.save()  # update updated_at
+
+
+def send_telemetry(log):
+    """Send anonymous telemetry during one window per week."""
+    try:
+        _send_telemetry(log)
+    except Exception as exc:
+        log.debug(f"Failed to send anonyomous stats: {exc}")

codex/librarian/telemeter/time.py

@@ -0,0 +1,61 @@
+"""Get telemeter send time."""
+
+from datetime import datetime, timedelta, timezone
+from uuid import UUID
+
+from codex.librarian.telemeter.telemeter import get_telemeter_timestamp
+from codex.models.admin import AdminFlag
+
+# Timing
+_ONE_DAY = 24 * 60 * 60
+_SECS_PER_WEEK = 7 * _ONE_DAY
+_MAX_UUID = 2**128
+_UUID_DIVISOR = _MAX_UUID / _SECS_PER_WEEK
+
+
+def _get_utc_start_of_week():
+    """Get timestamp for this Monady 00:00:00."""
+    # Monday, now o'clock.
+    utc_now = datetime.now(tz=timezone.utc)
+    start_of_week = utc_now - timedelta(days=utc_now.weekday())
+    # Monday, midnight.
+    return start_of_week.replace(hour=0, minute=0, second=0, microsecond=0).astimezone(
+        tz=timezone.utc
+    )
+
+
+def _is_created_recently(ts):
+    """Don't send if created recently."""
+    now = datetime.now(tz=timezone.utc)
+    since = ts.created_at - now
+    return abs(since.total_seconds()) < _ONE_DAY
+
+
+def _get_scheduled_time(ts):
+    """Compute the time of week to send from the uuid."""
+    start_of_week = _get_utc_start_of_week()
+    uuid = UUID(ts.version)
+    seconds_after_week_start = uuid.int / _UUID_DIVISOR
+    time_of_week = timedelta(seconds=seconds_after_week_start)
+    telemeter_time = start_of_week + time_of_week
+    telemeter_time = telemeter_time.astimezone(tz=timezone.utc)
+    if telemeter_time < ts.updated_at:
+        # Already ran this week, or created this week after run time.
+        telemeter_time = 0
+    return telemeter_time
+
+
+def get_telemeter_time(log):
+    """Get the time to send telemetry."""
+    # Should we schedule telemeter at all?
+    if not AdminFlag.objects.get(key=AdminFlag.FlagChoices.SEND_TELEMETRY.value).on:
+        log.debug("Telemeter disabled. Not scheduled.")
+        return 0
+
+    ts = get_telemeter_timestamp()
+
+    if _is_created_recently(ts):
+        log.debug("Telemeter created recently. Not scheduled.")
+        return 0
+
+    return _get_scheduled_time(ts)

codex/librarian/watchdog/db_snapshot.py

@@ -58,7 +58,7 @@ class CodexDatabaseSnapshot(DirectorySnapshot, LoggerBaseMixin):
         i = (st.st_ino, st.st_dev)
         self._inode_to_path[i] = path
 
-    def __init__(  # noqa: PLR0913
+    def __init__(
         self,
         path,
         _recursive=True,  # unused, always recursive

codex/librarian/watchdog/dir_snapshot_diff.py

@@ -21,9 +21,9 @@ class SnapshotDiffData:
 class CodexDirectorySnapshotDiff(DirectorySnapshotDiff):
     """Custom Diff allows inode only changes to be 'modified'."""
 
-    def _get_inode(self, directory, full_path):
+    def _get_inode(self, snapshot, full_path):
         """Get inode optionally ignoring device stats."""
-        result = directory.inode(full_path)
+        result = snapshot.inode(full_path)
         if self._ignore_device:
             result = result[0]
         return result
@@ -33,7 +33,7 @@ class CodexDirectorySnapshotDiff(DirectorySnapshotDiff):
         return self._get_inode(data.ref, path) == self._get_inode(data.snapshot, path)
 
     def _is_stats_equal(self, data, old_path, new_path):
-        """Return if the mtimze and size are equal.
+        """Return if the mtime and size are equal.
 
         For old paths in the ref and new paths in the snapshot.
         """
@@ -48,25 +48,27 @@ class CodexDirectorySnapshotDiff(DirectorySnapshotDiff):
                 if self._inode_only_modified:
                     data.modified.add(path)
                 else:
+                    # Add these to created and deleted. But...
+                    # The _find_moved_paths() inode tracker should
+                    #  sort them out.
+                    # This is the same as in upstream dirsnapshot.py
                     data.created.add(path)
                     data.deleted.add(path)
 
     def _find_moved_paths(self, data):
-        """Find moved paths."""
-        for path in tuple(data.deleted):
-            inode = data.ref.inode(path)
-            new_path = data.snapshot.path(inode)
-            if new_path:
+        """Find moved paths in deleted and created."""
+        for old_path in tuple(data.deleted):
+            inode = self._get_inode(data.ref, old_path)
+            if new_path := data.snapshot.path(inode):
                 # file is not deleted but moved
-                data.deleted.remove(path)
-                data.moved.add((path, new_path))
+                data.deleted.remove(old_path)
+                data.moved.add((old_path, new_path))
 
-        for path in tuple(data.created):
-            inode = data.snapshot.inode(path)
-            old_path = data.ref.path(inode)
-            if old_path:
-                data.created.remove(path)
-                data.moved.add((old_path, path))
+        for new_path in tuple(data.created):
+            inode = self._get_inode(data.snapshot, new_path)
+            if old_path := data.ref.path(inode):
+                data.created.remove(new_path)
+                data.moved.add((old_path, new_path))
 
     def _find_modified_paths(self, data):
         """Find modified paths."""
@@ -80,7 +82,7 @@ class CodexDirectorySnapshotDiff(DirectorySnapshotDiff):
         # Check moved paths for modificiations.
         for old_path, new_path in data.moved:
             if not self._is_stats_equal(data, old_path, new_path):
-                data.modified.add(old_path)
+                data.modified.add(new_path)
 
     def __init__(  # C901, PLR0912
         self, ref, snapshot, ignore_device=False, inode_only_modified=False

codex/librarian/watchdog/emitter.py

@@ -25,20 +25,18 @@ from codex.models import Library
 from codex.status import Status
 from codex.worker_base import WorkerBaseMixin
 
-_CODEX_EVENT_FILTER = frozenset(
-    {
-        FileMovedEvent,
-        FileModifiedEvent,
-        FileCreatedEvent,
-        FileDeletedEvent,
-        # FileClosedEvent,
-        # FileOpenedEvent,
-        DirMovedEvent,
-        DirModifiedEvent,
-        DirDeletedEvent,
-        # DirCreatedEvent,
-    }
-)
+_CODEX_EVENT_FILTER = [
+    FileMovedEvent,
+    FileModifiedEvent,
+    FileCreatedEvent,
+    FileDeletedEvent,
+    # FileClosedEvent,
+    # FileOpenedEvent,
+    DirMovedEvent,
+    DirModifiedEvent,
+    DirDeletedEvent,
+    # DirCreatedEvent,
+]
 _DOCKER_UNMOUNTED_FN = "DOCKER_UNMOUNTED_VOLUME"
 
 

codex/librarian/watchdog/events.py

@@ -234,7 +234,7 @@ class CodexCustomCoverEventHandler(CodexEventHandlerBase):
         try:
             if event.is_directory or event.event_type in self.IGNORED_EVENTS:
                 return
-            src_cover_match = self._match_group_cover_image(event.src_path)
+            src_cover_match = self._match_group_cover_image(str(event.src_path))
             if event.event_type == EVENT_TYPE_MOVED:
                 send_event = self._transform_event_moved(event, src_cover_match)
             elif src_cover_match:

codex/librarian/watchdog/observers.py

@@ -110,15 +110,15 @@ class UatuMixin(BaseObserver, WorkerBaseMixin):
         https://pythonhosted.org/watchdog/_modules/watchdog/observers/api.html#BaseObserver
         """
         if self._emitter_class != DatabasePollingEmitter:
-            return super().schedule(event_handler, path, recursive)
+            return super().schedule(event_handler, path, recursive=recursive)
         with self._lock:
-            watch = ObservedWatch(path, recursive)
+            watch = ObservedWatch(path, recursive=recursive)
             self._add_handler_for_watch(event_handler, watch)
 
             # If we don't have an emitter for this watch already, create it.
             if self._emitter_for_watch.get(watch) is None:
                 covers_only = isinstance(event_handler, CodexCustomCoverEventHandler)
-                emitter = self._emitter_class(
+                emitter = DatabasePollingEmitter(
                     event_queue=self.event_queue,
                     watch=watch,
                     timeout=self.timeout,
@@ -166,7 +166,7 @@ class LibraryPollingObserver(UatuMixin):
 
             for emitter in self.emitters:
                 if emitter.watch.path in paths:
-                    emitter.poll(force)
+                    emitter.poll(force)  # type: ignore
         except Exception:
             self.log.exception(
                 f"{self.__class__.__name__}.poll({library_pks}, {force})"

codex/migrations/0028_telemeter.py

@@ -0,0 +1,48 @@
+"""Generated by Django 5.0.6 on 2024-07-02 21:01."""
+
+from typing import ClassVar
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    """Migrate db."""
+
+    dependencies: ClassVar[list] = [  # type: ignore
+        ("codex", "0027_import_order_and_covers"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="adminflag",
+            name="key",
+            field=models.CharField(
+                choices=[
+                    ("FV", "Folder View"),
+                    ("RG", "Registration"),
+                    ("NU", "Non Users"),
+                    ("AU", "Auto Update"),
+                    ("SO", "Search Index Optimize"),
+                    ("IM", "Import Metadata"),
+                    ("ST", "Send Telemetry"),
+                ],
+                db_index=True,
+                max_length=2,
+            ),
+        ),
+        migrations.AlterField(
+            model_name="timestamp",
+            name="key",
+            field=models.CharField(
+                choices=[
+                    ("JA", "Janitor"),
+                    ("VR", "Codex Version"),
+                    ("SI", "Search Index UUID"),
+                    ("AP", "API Key"),
+                    ("TS", "Telemeter Sent"),
+                ],
+                db_index=True,
+                max_length=2,
+            ),
+        ),
+    ]

codex/models/admin.py

@@ -39,6 +39,7 @@ class AdminFlag(BaseModel):
         AUTO_UPDATE = "AU"
         SEARCH_INDEX_OPTIMIZE = "SO"
         IMPORT_METADATA = "IM"
+        SEND_TELEMETRY = "ST"
 
     FALSE_DEFAULTS = frozenset({FlagChoices.AUTO_UPDATE})
 
@@ -86,6 +87,7 @@ class Timestamp(BaseModel):
         CODEX_VERSION = "VR", _("Codex Version")
         SEARCH_INDEX_UUID = "SI", _("Search Index UUID")
         API_KEY = "AP", _("API Key")
+        TELEMETER_SENT = "TS", _("Telemeter Sent")
 
     key = CharField(db_index=True, max_length=2, choices=TimestampChoices.choices)
     version = CharField(max_length=MAX_FIELD_LEN, default="")

codex/models/comic.py

@@ -246,7 +246,7 @@ class Comic(WatchedPathBrowserGroup):
         return issue_str
 
     @classmethod
-    def get_title(  # noqa: PLR0913
+    def get_title(
         cls, obj, volume=True, zero_pad=None, name=True, filename_fallback=False
     ):
         """Create the comic title for display."""

codex/search/backend.py

@@ -122,7 +122,10 @@ class CodexSearchBackend(CodexBackendSearchMixin, WhooshSearchBackend, WorkerBas
             "created_at": ("created",),
             "critical_rating": gen_multipart_field_aliases("critical_rating"),
             "genres": ("genre",),
-            "file_type": ("type",),
+            "file_type": (
+                "filetype",
+                "type",
+            ),
             "identifier": ("id", "nss"),
             "identifier_type": ("idtype", "id_type", "nid"),
             "issue_number": ("number",),
@@ -132,7 +135,11 @@ class CodexSearchBackend(CodexBackendSearchMixin, WhooshSearchBackend, WorkerBas
             "original_format": ("format",),
             "page_count": ("pages",),
             "reading_direction": ("direction",),
-            "search_path": ("path", *gen_multipart_field_aliases("folders")),
+            "search_path": (
+                "path",
+                "filename",
+                *gen_multipart_field_aliases("folders"),
+            ),
             "series_groups": gen_multipart_field_aliases("series_groups"),
             "scan_info": ("scan",),
             "stories": ("story",),
@@ -431,7 +438,7 @@ class CodexSearchBackend(CodexBackendSearchMixin, WhooshSearchBackend, WorkerBas
             raise
         return count
 
-    def update(  # noqa: PLR0913, # type: ignore
+    def update(  # type: ignore
         self,
         index,
         iterable,

codex/serializers/browser/mixins.py

@@ -53,8 +53,7 @@ class BrowserAggregateSerializerMixin(Serializer):
                 )
                 continue
 
-            if dt > mtime:
-                mtime = dt
+            mtime = max(dt, mtime)
 
         if obj.group != COMIC_GROUP and (
             mbua := getattr(obj, "max_bookmark_updated_at", None)