codex 1.4.1__py3-none-any.whl → 1.4.3__py3-none-any.whl

codex/applications/websocket.py

@@ -18,7 +18,7 @@ WEBSOCKET_APPLICATION = AllowedHostsOriginValidator(
             [
                 path(
                     f"{ROOT_PREFIX}api/v3/ws",
-                    NotifierConsumer.as_asgi(),  # type: ignore
+                    NotifierConsumer.as_asgi(),
                     name="websocket",
                 ),
             ]

codex/librarian/importer/aggregate_metadata.py

@@ -1,5 +1,6 @@
 """Aggregate metadata from comics to prepare for importing."""
 from pathlib import Path
+from types import MappingProxyType
 from zipfile import BadZipFile
 
 from comicbox.comic_archive import ComicArchive
@@ -20,9 +21,11 @@ class AggregateMetadataMixin(CleanMetadataMixin):
 
     _BROWSER_GROUPS = (Publisher, Imprint, Series, Volume)
     _BROWSER_GROUP_TREE_COUNT_FIELDS = frozenset(["volume_count", "issue_count"])
-    _GROUP_TREES_INIT = {
-        "group_trees": {Publisher: {}, Imprint: {}, Series: {}, Volume: {}},
-    }
+    _GROUP_TREES_INIT = MappingProxyType(
+        {
+            "group_trees": {Publisher: {}, Imprint: {}, Series: {}, Volume: {}},
+        }
+    )
     _AGGREGATE_COMICBOX_CONFIG = AttrDict({**COMICBOX_CONFIG, "close_fd": False})
 
     @staticmethod
@@ -36,6 +39,36 @@ class AggregateMetadataMixin(CleanMetadataMixin):
                 file_type = suffix
         return file_type
 
+    @classmethod
+    def _get_group_tree(cls, md):
+        """Create the group tree to counts map for a single comic."""
+        # Create group tree
+        group_tree = []
+        for group_cls in cls._BROWSER_GROUPS:
+            group_field = group_cls.__name__.lower()
+            # some volumes are read by ComicArchive as ints, cast
+            group_name = str(md.get(group_field, Publisher.DEFAULT_NAME))
+            # This fixes no imprint or whatever being in md
+            md[group_field] = group_name
+            group_tree.append(group_name)
+
+        # Add counts to group tree.
+        groups_md = {}
+        md_group_count_fields = cls._BROWSER_GROUP_TREE_COUNT_FIELDS & md.keys()
+        for key in md_group_count_fields:
+            groups_md[key] = md.pop(key)
+        return {tuple(group_tree): groups_md}
+
+    @staticmethod
+    def _get_m2m_metadata(md, path):
+        """Many_to_many fields get moved into a separate dict."""
+        m2m_md = {}
+        md_m2m_fields = COMIC_M2M_FIELD_NAMES & md.keys()
+        for field in md_m2m_fields:
+            m2m_md[field] = md.pop(field)
+        m2m_md["folders"] = Path(path).parents
+        return m2m_md
+
     def _get_path_metadata(self, path):
         """Get the metatada from comicbox and munge it a little."""
         md = {}
@@ -50,28 +83,8 @@ class AggregateMetadataMixin(CleanMetadataMixin):
             md["path"] = path
             md = self.clean_md(md)
 
-            # Create group tree
-            group_tree = []
-            for group_cls in self._BROWSER_GROUPS:
-                group_field = group_cls.__name__.lower()
-                # some volumes are read by ComicArchive as ints, cast
-                group_name = str(md.get(group_field, Publisher.DEFAULT_NAME))
-                # This fixes no imprint or whatever being in md
-                md[group_field] = group_name
-                group_tree.append(group_name)
-
-            # Add counts to group tree.
-            groups_md = {}
-            md_group_count_fields = self._BROWSER_GROUP_TREE_COUNT_FIELDS & md.keys()
-            for key in md_group_count_fields:
-                groups_md[key] = md.pop(key)
-            group_tree_md[tuple(group_tree)] = groups_md
-
-            # Many_to_many fields get moved into a separate dict
-            md_m2m_fields = COMIC_M2M_FIELD_NAMES & md.keys()
-            for field in md_m2m_fields:
-                m2m_md[field] = md.pop(field)
-            m2m_md["folders"] = Path(path).parents
+            group_tree_md = self._get_group_tree(md)
+            m2m_md = self._get_m2m_metadata(md, path)
 
         except (UnsupportedArchiveTypeError, BadRarFile, BadZipFile, OSError) as exc:
             self.log.warning(f"Failed to import {path}: {exc}")

codex/librarian/importer/clean_metadata.py

@@ -2,7 +2,7 @@
 import re
 from contextlib import suppress
 from decimal import Decimal
-from typing import Any, Optional
+from typing import Any, Optional, Union
 
 from comicbox.metadata.comic_base import ComicBaseMetadata
 from django.db.models.fields import CharField, DecimalField, PositiveSmallIntegerField
@@ -54,7 +54,6 @@ _MD_CHAR_KEYS = frozenset(
 _TWO_PLACES = Decimal("0.01")
 _PSI_MAX = 2147483647
 _GROUPS = frozenset(("publisher", "imprint", "series", "volume"))
-_URL_MAX_LENGTH = 200
 _M2M_NAMED_KEYS = frozenset(
     (
         "characters",
@@ -74,10 +73,21 @@ class CleanMetadataMixin(QueuedThread):
     """Clean metadata before importing."""
 
     @staticmethod
-    def _clean_decimal(value, field_name: str):
+    def _clean_string(value: Union[bytes, str]):
+        """Replace unstorable, unprintable characters from metadata."""
+        # https://stackoverflow.com/questions/27366479/python-3-os-walk-file-paths-unicodeencodeerror-utf-8-codec-cant-encode-s
+        if isinstance(value, str):
+            value = value.encode("utf8", "replace")
+
+        return value.decode("utf8", "replace")
+
+    @classmethod
+    def _clean_decimal(cls, value, field_name: str):
         field: DecimalField = Comic._meta.get_field(field_name)  # type: ignore
         try:
             # Comicbox now gives issues as strings, convert them to decimal here.
+            if isinstance(value, (str, bytes)):
+                value = cls._clean_string(value)
             value = ComicBaseMetadata.parse_decimal(value)
             value = value.quantize(_TWO_PLACES)
             value = value.max(_DECIMAL_ZERO)
@@ -89,7 +99,9 @@ class CleanMetadataMixin(QueuedThread):
 
     def _parse_comic_issue(self, md: dict[str, Any]):
         """Parse the issue field."""
-        issue_str = md.get("issue", "").strip()
+        issue_str = md.get("issue", "")
+        issue_str = self._clean_string(issue_str)
+        issue_str = issue_str.strip()
         try:
             match = _PARSE_ISSUE_MATCHER.match(issue_str)
             issue, issue_suffix = match.groups()  # type: ignore
@@ -151,13 +163,14 @@ class CleanMetadataMixin(QueuedThread):
         with suppress(KeyError):
             md["name"] = md.pop("title")
 
-    @staticmethod
-    def _clean_charfield(value: Optional[str], field: CharField) -> Optional[str]:
+    @classmethod
+    def _clean_charfield(cls, value: Optional[str], field: CharField) -> Optional[str]:
         try:
             if value is None:
                 raise ValueError  # noqa TRY301
             value = str(value)
             value = value[: field.max_length].strip()
+            value = cls._clean_string(value)
         except Exception:
             value = None if field.null else ""
         return value
@@ -202,14 +215,14 @@ class CleanMetadataMixin(QueuedThread):
                     good_story_arc_numbers[good_story_arc_name] = number
             md["story_arc_numbers"] = good_story_arc_numbers
 
-    @staticmethod
-    def _clean_comic_web(md):
+    @classmethod
+    def _clean_comic_web(cls, md):
         """URL field is a special charfield."""
-        if "web" not in md:
-            return
-        try:
-            md["web"] = md["web"][:_URL_MAX_LENGTH]
-        except Exception:
+        web = md.get("web")
+        if web:
+            field: CharField = Comic._meta.get_field("web")  # type:ignore
+            web = cls._clean_charfield(web, field)
+        if not web and web in md:
             del md["web"]
 
     @classmethod

codex/librarian/importer/create_fks.py

@@ -161,7 +161,7 @@ class CreateForeignKeysMixin(QueuedThread):
                     parent = Folder.objects.get(path=parent_path)
                 except Folder.DoesNotExist:
                     if path.parent != Path(library.path):
-                        self.log.exception(
+                        self.log.warning(
                             f"Can't find parent folder {parent_path}"
                             f" for {path} in library {library.path}"
                         )

codex/librarian/librariand.py

@@ -2,6 +2,7 @@
 from collections import namedtuple
 from multiprocessing import Manager, Process
 from threading import active_count
+from types import MappingProxyType
 
 from caseconverter import snakecase
 from comicbox.comic_archive import ComicArchive
@@ -53,10 +54,12 @@ class LibrarianDaemon(Process, LoggerBaseMixin):
         LibraryPollingObserver,
         JanitorThread,
     )
-    _THREAD_CLASS_MAP = {
-        snakecase(thread_class.__name__): thread_class
-        for thread_class in _THREAD_CLASSES
-    }
+    _THREAD_CLASS_MAP = MappingProxyType(
+        {
+            snakecase(thread_class.__name__): thread_class
+            for thread_class in _THREAD_CLASSES
+        }
+    )
     LibrarianThreads = namedtuple("LibrarianThreads", _THREAD_CLASS_MAP.keys())
 
     proc = None

codex/librarian/watchdog/db_snapshot.py

@@ -41,7 +41,7 @@ class CodexDatabaseSnapshot(DirectorySnapshot, LoggerBaseMixin):
                     f"Force delete missing path with missing db stat: {path}"
                 )
                 # This will trigger a deleted event
-                stat = Comic.ZERO_STAT
+                stat = list(Comic.ZERO_STAT)
 
         if force:
             # Fake mtime will trigger modified event

codex/librarian/watchdog/event_batcherd.py

@@ -7,6 +7,7 @@ and then re-serialize everything in this batcher and the event Handler
 """
 from contextlib import suppress
 from copy import deepcopy
+from types import MappingProxyType
 
 from watchdog.events import EVENT_TYPE_MOVED
 
@@ -19,16 +20,18 @@ class WatchdogEventBatcherThread(AggregateMessageQueuedThread):
     """Batch watchdog events into bulk database tasks."""
 
     CLS_SUFFIX = -len("Event")
-    DBDIFF_TASK_PARAMS = {
-        "library_id": None,
-        "dirs_moved": {},
-        "files_moved": {},
-        "files_modified": set(),
-        "files_created": set(),
-        "dirs_deleted": set(),
-        "dirs_modified": set(),
-        "files_deleted": set(),
-    }
+    DBDIFF_TASK_PARAMS = MappingProxyType(
+        {
+            "library_id": None,
+            "dirs_moved": {},
+            "files_moved": {},
+            "files_modified": set(),
+            "files_created": set(),
+            "dirs_deleted": set(),
+            "dirs_modified": set(),
+            "files_deleted": set(),
+        }
+    )
     MAX_DELAY = 60
     MAX_ITEMS_PER_GB = 50000
 
@@ -42,7 +45,7 @@ class WatchdogEventBatcherThread(AggregateMessageQueuedThread):
     def _ensure_library_args(self, library_id):
         if library_id in self.cache:
             return
-        args = deepcopy(self.DBDIFF_TASK_PARAMS)
+        args = deepcopy(dict(self.DBDIFF_TASK_PARAMS))
         args["library_id"] = library_id
         self.cache[library_id] = args
 

codex/librarian/watchdog/events.py

@@ -19,7 +19,7 @@ from codex.logger_base import LoggerBaseMixin
 class CodexLibraryEventHandler(FileSystemEventHandler, LoggerBaseMixin):
     """Handle watchdog events for comics in a library."""
 
-    IGNORED_EVENTS = {EVENT_TYPE_CLOSED, EVENT_TYPE_OPENED}
+    IGNORED_EVENTS = frozenset((EVENT_TYPE_CLOSED, EVENT_TYPE_OPENED))
 
     def __init__(self, library, *args, **kwargs):
         """Let us send along he library id."""

codex/logger/loggerd.py

@@ -1,6 +1,7 @@
 """Central Logging Thread."""
 import logging
 from logging.handlers import QueueListener, RotatingFileHandler
+from types import MappingProxyType
 
 from colors import color
 
@@ -10,28 +11,30 @@ from codex.settings.settings import DEBUG, LOG_DIR, LOG_TO_CONSOLE, LOG_TO_FILE
 class ColorFormatter(logging.Formatter):
     """Logging Formatter to add colors and count warning / errors."""
 
-    FORMAT_COLORS = {
-        "CRITICAL": {"fg": "red", "style": "bold"},
-        "ERROR": {"fg": "red"},
-        "WARNING": {"fg": "yellow"},
-        "INFO": {"fg": "green"},
-        "DEBUG": {"fg": "black", "style": "bold"},
-        "NOTSET": {"fg": "blue"},
-    }
-    FORMATTERS = {}
+    FORMAT_COLORS = MappingProxyType(
+        {
+            "CRITICAL": {"fg": "red", "style": "bold"},
+            "ERROR": {"fg": "red"},
+            "WARNING": {"fg": "yellow"},
+            "INFO": {"fg": "green"},
+            "DEBUG": {"fg": "black", "style": "bold"},
+            "NOTSET": {"fg": "blue"},
+        }
+    )
 
     def __init__(self, fmt, **kwargs):
         """Set up the FORMATS dict."""
         super().__init__(**kwargs)
+        self.formatters = {}
         for level_name, args in self.FORMAT_COLORS.items():
             levelno = getattr(logging, level_name)
             template = color(fmt, **args)
             formatter = logging.Formatter(fmt=template, **kwargs)
-            self.FORMATTERS[levelno] = formatter
+            self.formatters[levelno] = formatter
 
     def format(self, record):  # noqa A003
         """Format each log message."""
-        formatter = self.FORMATTERS[record.levelno]
+        formatter = self.formatters[record.levelno]
         return formatter.format(record)
 
 

codex/migrations/0001_init.py

@@ -2,6 +2,7 @@
 
 import datetime
 from decimal import Decimal
+from typing import ClassVar
 
 import django.db.models.deletion
 from django.conf import settings
@@ -15,12 +16,12 @@ class Migration(migrations.Migration):
 
     initial = True
 
-    dependencies = [
+    dependencies: ClassVar[list] = [
         ("sessions", "0001_initial"),
         migrations.swappable_dependency(settings.AUTH_USER_MODEL),
     ]
 
-    operations = [
+    operations: ClassVar[list] = [
         migrations.CreateModel(
             name="Character",
             fields=[

codex/migrations/0002_auto_20200826_0622.py

@@ -1,16 +1,18 @@
 """Generated by Django 3.1 on 2020-08-26 06:22."""
 
+from typing import ClassVar
+
 from django.db import migrations
 
 
 class Migration(migrations.Migration):
     """Change libraries verbose name."""
 
-    dependencies = [
+    dependencies: ClassVar[list] = [
         ("codex", "0001_init"),
     ]
 
-    operations = [
+    operations: ClassVar[list] = [
         migrations.AlterModelOptions(
             name="library",
             options={"verbose_name_plural": "libraries"},

codex/migrations/0003_auto_20200831_2033.py

@@ -1,5 +1,7 @@
 """Generated by Django 3.1 on 2020-08-31 20:33."""
 
+from typing import ClassVar
+
 import django.db.models.deletion
 from django.db import migrations, models
 
@@ -7,11 +9,11 @@ from django.db import migrations, models
 class Migration(migrations.Migration):
     """Credit roles can be none."""
 
-    dependencies = [
+    dependencies: ClassVar[list] = [
         ("codex", "0002_auto_20200826_0622"),
     ]
 
-    operations = [
+    operations: ClassVar[list] = [
         migrations.AlterField(
             model_name="credit",
             name="role",

codex/migrations/0004_failedimport.py

@@ -1,5 +1,7 @@
 """Generated by Django 3.1.1 on 2020-09-14 22:15."""
 
+from typing import ClassVar
+
 import django.db.models.deletion
 from django.db import migrations, models
 
@@ -7,11 +9,11 @@ from django.db import migrations, models
 class Migration(migrations.Migration):
     """Keep track of failed imports."""
 
-    dependencies = [
+    dependencies: ClassVar[list] = [
         ("codex", "0003_auto_20200831_2033"),
     ]
 
-    operations = [
+    operations: ClassVar[list] = [
         migrations.CreateModel(
             name="FailedImport",
             fields=[

codex/migrations/0005_auto_20200918_0146.py

@@ -1,16 +1,18 @@
 """Generated by Django 3.1.1 on 2020-09-18 01:46."""
 
+from typing import ClassVar
+
 from django.db import migrations
 
 
 class Migration(migrations.Migration):
     """Update verbose names."""
 
-    dependencies = [
+    dependencies: ClassVar[list] = [
         ("codex", "0004_failedimport"),
     ]
 
-    operations = [
+    operations: ClassVar[list] = [
         migrations.AlterModelOptions(
             name="comic",
             options={"verbose_name": "Issue"},

codex/migrations/0006_update_default_names_and_remove_duplicate_comics.py

@@ -1,13 +1,18 @@
 """Generated by Django 3.2.9 on 2021-11-04 03:03."""
 
+from types import MappingProxyType
+from typing import ClassVar
+
 from django.db import migrations
 from django.db.models.functions import Now
 
-MODEL_NAMES = {
-    "Series": "Default Series",
-    "Imprint": "Main Imprint",
-    "Publisher": "No Publisher",
-}
+MODEL_NAMES = MappingProxyType(
+    {
+        "Series": "Default Series",
+        "Imprint": "Main Imprint",
+        "Publisher": "No Publisher",
+    }
+)
 NEW_DEFAULT_NAME = ""
 UPDATE_FIELDS = ("stat", "updated_at")
 
@@ -58,11 +63,11 @@ def remove_duplicate_comics(apps, _schema_editor):
 class Migration(migrations.Migration):
     """Change default names to ''."""
 
-    dependencies = [
+    dependencies: ClassVar[list] = [
         ("codex", "0005_auto_20200918_0146"),
     ]
 
-    operations = [
+    operations: ClassVar[list] = [
         migrations.RunPython(update_default_names),
         migrations.RunPython(remove_duplicate_comics),
     ]

codex/migrations/0007_auto_20211210_1710.py

@@ -1,6 +1,7 @@
 """Generated by Django 3.2.9 on 2021-12-11 01:10."""
 
 import datetime
+from typing import ClassVar
 
 import django.db.models.deletion
 from django.db import migrations, models
@@ -9,11 +10,11 @@ from django.db import migrations, models
 class Migration(migrations.Migration):
     """Large migration for v0.7.0."""
 
-    dependencies = [
+    dependencies: ClassVar[list] = [
         ("codex", "0006_update_default_names_and_remove_duplicate_comics"),
     ]
 
-    operations = [
+    operations: ClassVar[list] = [
         migrations.CreateModel(
             name="LatestVersion",
             fields=[

codex/migrations/0008_alter_comic_created_at_alter_comic_format_and_more.py

@@ -1,5 +1,7 @@
 """Generated by Django 4.0 on 2021-12-17 04:36."""
 
+from typing import ClassVar
+
 import django.db.models.deletion
 from django.db import migrations, models
 
@@ -7,11 +9,11 @@ from django.db import migrations, models
 class Migration(migrations.Migration):
     """Indexes for filtered and sorted comic fields."""
 
-    dependencies = [
+    dependencies: ClassVar[list] = [
         ("codex", "0007_auto_20211210_1710"),
     ]
 
-    operations = [
+    operations: ClassVar[list] = [
         migrations.AlterField(
             model_name="comic",
             name="created_at",

codex/migrations/0009_alter_comic_parent_folder.py

@@ -1,5 +1,7 @@
 """Generated by Django 4.0 on 2021-12-19 17:47."""
 
+from typing import ClassVar
+
 import django.db.models.deletion
 from django.db import migrations, models
 
@@ -7,11 +9,11 @@ from django.db import migrations, models
 class Migration(migrations.Migration):
     """Override related_in becauese interferes with comic.folders."""
 
-    dependencies = [
+    dependencies: ClassVar[list] = [
         ("codex", "0008_alter_comic_created_at_alter_comic_format_and_more"),
     ]
 
-    operations = [
+    operations: ClassVar[list] = [
         migrations.AlterField(
             model_name="comic",
             name="parent_folder",

codex/migrations/0010_haystack.py

@@ -1,5 +1,7 @@
 """Generated by Django 4.0.1 on 2022-01-16 05:31."""
 
+from typing import ClassVar
+
 import django.db.models.deletion
 from django.db import migrations, models
 
@@ -7,11 +9,11 @@ from django.db import migrations, models
 class Migration(migrations.Migration):
     """Haystack search engine."""
 
-    dependencies = [
+    dependencies: ClassVar[list] = [
         ("codex", "0009_alter_comic_parent_folder"),
     ]
 
-    operations = [
+    operations: ClassVar[list] = [
         migrations.CreateModel(
             # Fake model. Not managed.
             name="QueueJob",

codex/migrations/0011_library_groups_and_metadata_changes.py

@@ -1,5 +1,6 @@
 """Generated by Django 4.0.1 on 2022-01-31 22:09 & tweaked by aj."""
 from decimal import Decimal
+from typing import ClassVar
 
 from django.db import migrations, models
 
@@ -26,12 +27,12 @@ def critical_rating_to_decimal(apps, _schema_editor):
 class Migration(migrations.Migration):
     """Library group ACLS and metadata changes."""
 
-    dependencies = [
+    dependencies: ClassVar[list] = [
         ("auth", "0012_alter_user_first_name_max_length"),
         ("codex", "0010_haystack"),
     ]
 
-    operations = [
+    operations: ClassVar[list] = [
         migrations.AddField(
             model_name="comic",
             name="community_rating",

codex/migrations/0012_rename_description_comic_comments.py

@@ -1,15 +1,17 @@
 """Generated by Django 4.0.2 on 2022-02-24 20:58."""
+from typing import ClassVar
+
 from django.db import migrations
 
 
 class Migration(migrations.Migration):
     """Rename comic description to comic comments."""
 
-    dependencies = [
+    dependencies: ClassVar[list] = [
         ("codex", "0011_library_groups_and_metadata_changes"),
     ]
 
-    operations = [
+    operations: ClassVar[list] = [
         migrations.RenameField(
             model_name="comic",
             old_name="description",

codex/migrations/0013_int_issue_count_longer_charfields.py

@@ -1,6 +1,6 @@
 """Generated by Django 4.0.2 on 2022-03-25 23:16."""
-
 from decimal import Decimal
+from typing import ClassVar
 
 from django.db import migrations, models
 
@@ -28,11 +28,11 @@ def cast_issue_count(apps, _schema_editor):
 class Migration(migrations.Migration):
     """Larger valid fields."""
 
-    dependencies = [
+    dependencies: ClassVar[list] = [
         ("codex", "0012_rename_description_comic_comments"),
     ]
 
-    operations = [
+    operations: ClassVar[list] = [
         migrations.RenameField(
             model_name="volume", old_name="issue_count", new_name="issue_count_decimal"
         ),