codex 1.5.19__py3-none-any.whl → 1.6.0a0__py3-none-any.whl

codex/librarian/importer/query_fks.py

@@ -11,11 +11,11 @@ from codex.librarian.importer.const import (
     CONTRIBUTORS_FIELD_NAME,
     COUNT_FIELDS,
     DICT_MODEL_CLASS_FIELDS_MAP,
+    DICT_MODEL_FIELD_MODEL_MAP,
     DICT_MODEL_REL_MAP,
     GROUP_COMPARE_FIELDS,
     GROUP_TREES,
     IDENTIFIERS_FIELD_NAME,
-    IDENTIFIERS_MODEL_REL_MAP,
     IMPRINT,
     PARENT_FOLDER,
     PUBLISHER,
@@ -41,16 +41,21 @@ from codex.threads import QueuedThread
 
 _CREATE_GROUPS = "create_groups"
 _UPDATE_GROUPS = "update_groups"
-_CLASS_QUERY_FIELDS_MAP = {
-    Contributor: ("role__name", "person__name"),
-    StoryArcNumber: ("story_arc__name", "number"),
-    Folder: ("path",),
-    Imprint: ("publisher__name", "name"),
-    Series: ("publisher__name", "imprint__name", "name"),
-    Volume: ("publisher__name", "imprint__name", "series__name", "name"),
-    Identifier: ("identifier_type__name", "nss", "url"),
-}
+_CLASS_QUERY_FIELDS_MAP = MappingProxyType(
+    {
+        Contributor: ("role__name", "person__name"),
+        StoryArcNumber: ("story_arc__name", "number"),
+        Folder: ("path",),
+        Imprint: ("publisher__name", "name"),
+        Series: ("publisher__name", "imprint__name", "name"),
+        Volume: ("publisher__name", "imprint__name", "series__name", "name"),
+        Identifier: ("identifier_type__name", "nss"),
+    }
+)
 _DEFAULT_QUERY_FIELDS = ("name",)
+_EXTRA_UPDATE_FIELDS_MAP = MappingProxyType(
+    {Series: ("volume_count",), Volume: ("issue_count",)}
+)
 
 LOG = get_logger(__name__)
 
@@ -71,24 +76,21 @@ class QueryForeignKeysMixin(QueuedThread):
         return fks_total
 
     @staticmethod
-    def _query_existing_mds(fk_cls, fk_filter):
+    def _query_existing_mds(fk_cls, fk_filter, extra_fields=None):
         """Query existing metatata tables."""
         fields = _CLASS_QUERY_FIELDS_MAP.get(fk_cls, _DEFAULT_QUERY_FIELDS)
+        if extra_fields:
+            fields += extra_fields
         flat = len(fields) == 1 and fk_cls != Publisher
-        # print(fk_filter)
-        qs = (
-            fk_cls.objects.filter(fk_filter)
-            .order_by("pk")
-            .values_list(*fields, flat=flat)
-        )
+        qs = fk_cls.objects.filter(fk_filter).values_list(*fields, flat=flat)
         return frozenset(qs)
 
     def _query_create_metadata(  # noqa: PLR0913
         self,
         fk_cls,
         create_mds,
+        update_mds,
         q_obj: Q,
-        q_and_chunk_prefix,
         status,
     ):
         """Get create metadata by comparing proposed meatada to existing rows."""
@@ -102,11 +104,16 @@ class QueryForeignKeysMixin(QueuedThread):
             # django.db.utils.OperationalError: Expression tree is too large (maximum depth 1000)
             children_chunk = q_obj.children[offset : offset + FILTER_BATCH_SIZE]
             filter_chunk = Q(*children_chunk, _connector=Q.OR)
-            if q_and_chunk_prefix:
-                filter_chunk = q_and_chunk_prefix & filter_chunk
 
             existing_mds = self._query_existing_mds(fk_cls, filter_chunk)
             create_mds.difference_update(existing_mds)
+            if extra_update_fields := _EXTRA_UPDATE_FIELDS_MAP.get(fk_cls):
+                # The mds that are in the existing_mds, but don't match the proposed mds with the extra update fields
+                update_mds.update(existing_mds)
+                match_with_extra_fields_mds = self._query_existing_mds(
+                    fk_cls, filter_chunk, extra_update_fields
+                )
+                update_mds.difference_update(match_with_extra_fields_mds)
 
             status.complete += len(filter_chunk)
             self.status_controller.update(status)
@@ -137,7 +144,9 @@ class QueryForeignKeysMixin(QueuedThread):
 
         filter_args[key] = group_name
 
-    def _get_create_group_set(self, groups, group_cls, create_group_set, status):
+    def _get_create_group_set(  # noqa: PLR0913
+        self, groups, group_cls, create_group_set, update_group_set, status
+    ):
         """Create the create group set."""
         all_filters = Q()
         for group_tree in groups:
@@ -153,10 +162,12 @@ class QueryForeignKeysMixin(QueuedThread):
             all_filters |= Q(**filter_args)
 
         candidate_groups = set(groups.keys())
+        update_groups = set()
         count = self._query_create_metadata(
-            group_cls, candidate_groups, all_filters, None, status
+            group_cls, candidate_groups, update_groups, all_filters, status
         )
         create_group_set.update(candidate_groups)
+        update_group_set.update(update_groups)
         return count
 
     @classmethod
@@ -174,6 +185,7 @@ class QueryForeignKeysMixin(QueuedThread):
         """Query missing groups for one group tree depth."""
         (
             create_group_set,
+            update_group_set,
             group_cls,
             create_and_update_groups,
             status,
@@ -185,7 +197,8 @@ class QueryForeignKeysMixin(QueuedThread):
                 group_tree,
                 count_value,
             )
-        elif group_cls in (Series, Volume):
+        elif group_cls in update_group_set:
+            # This always updates Volume or Series
             self._update_action_group(
                 group_cls,
                 create_and_update_groups[_UPDATE_GROUPS],
@@ -284,9 +297,18 @@ class QueryForeignKeysMixin(QueuedThread):
             status.subtitle = vnp
 
         create_group_set = set()
-        self._get_create_group_set(groups, group_cls, create_group_set, status)
+        update_group_set = set()
+        self._get_create_group_set(
+            groups, group_cls, create_group_set, update_group_set, status
+        )
 
-        data = (create_group_set, group_cls, create_and_update_groups, status)
+        data = (
+            create_group_set,
+            update_group_set,
+            group_cls,
+            create_and_update_groups,
+            status,
+        )
         for group_tree, count_value in groups.items():
             self._query_group_tree(data, group_tree, count_value)
 
@@ -317,43 +339,37 @@ class QueryForeignKeysMixin(QueuedThread):
     #########
     @staticmethod
     def _query_missing_identifiers_model_obj(
-        key_rel, value_rel_map, item, possible_create_objs
+        value_rel_map, key, values, possible_create_objs, url_restore_map
     ):
         """Update all_filters and possible_create_objs for identifiers."""
         # identifiers is unique at this time for having more than one value as a dict.
-        key, values = item
-        values_list = []
-        for val in values:
-            values_list.append(dict(val))
-
         value_filter = Q()
-        for values_dict in values_list:
+        # value filter
+        for value in values:
+            values_dict = dict(value)
+            create_obj = (key, values_dict.get("nss", ""))
+            url_restore_map[create_obj] = values_dict.get("url", "")
+            possible_create_objs.add(create_obj)
             value_sub_filter = Q()
             for value_key, obj_rel in value_rel_map.items():
-                value = values_dict.get(value_key)
-                value_sub_filter &= Q(**{f"{obj_rel}": value})
+                values_value = values_dict.get(value_key)
+                value_sub_filter &= Q(**{f"{obj_rel}": values_value})
             if value_sub_filter:
                 value_filter |= value_sub_filter
 
-        if not value_filter:
-            return value_filter, None
-
-        obj_filter = Q(**{key_rel: key})
-
-        for value in values:
-            possible_create_objs.add((key, value))
-        return value_filter, obj_filter
+        return value_filter
 
     @staticmethod
-    def _query_missing_dict_model_obj(key_rel, value_rel, item, possible_create_objs):
+    def _query_missing_dict_model_obj(value_rel, key, values, possible_create_objs, _):
         """Update all_filters and possible_create_objs for this obj."""
-        key, values = item
+        for value in values:
+            possible_create_objs.add((key, value))
 
         filter_isnull = None in values
         filter_values = values - {None}
 
         if not filter_values and not filter_isnull:
-            return Q(), None
+            return Q()
 
         value_filter = (
             Q(**{f"{value_rel}__in": filter_values}) if filter_values else Q()
@@ -361,58 +377,88 @@ class QueryForeignKeysMixin(QueuedThread):
         if filter_isnull:
             value_filter = value_filter | Q(**{f"{value_rel}__isnull": True})
 
-        obj_filter = Q(**{key_rel: key})
+        return value_filter
 
-        for value in values:
-            possible_create_objs.add((key, value))
-        return value_filter, obj_filter
+    def _query_missing_dict_model_add_to_query_filter_map(  # noqa: PLR0913
+        self,
+        key,
+        values,
+        field_name,
+        possible_create_objs,
+        query_filter_map,
+        url_restore_map,
+    ):
+        """Add value filter to query filter map."""
+        # Get the value filter
+        if field_name == "identifiers":
+            query_missing_method = self._query_missing_identifiers_model_obj
+        else:
+            query_missing_method = self._query_missing_dict_model_obj
+        key_rel, value_rel = DICT_MODEL_REL_MAP[field_name]
 
-    @staticmethod
-    def _add_value_filter_map(value_filter, filter_and_prefix, query_filter_map):
-        """Add a filter_and_prefix and the value filter to the map."""
-        if not value_filter:
-            return
+        value_filter = query_missing_method(
+            value_rel, key, values, possible_create_objs, url_restore_map
+        )
+
+        # Add value_filter to the query_filter_map
+        filter_and_prefix = Q(**{key_rel: key})
         if filter_and_prefix not in query_filter_map:
             query_filter_map[filter_and_prefix] = Q()
         query_filter_map[filter_and_prefix] |= value_filter
 
+    @staticmethod
+    def _query_missing_dict_model_identifiers_restore_urls(
+        field_name, possible_create_objs, url_restore_map
+    ):
+        """Restore urls to only the identifier create objs."""
+        if field_name != "identifiers":
+            return
+        restored_create_objs = []
+        for create_obj in possible_create_objs:
+            url = url_restore_map.get(create_obj)
+            restored_create_objs.append((*create_obj, url))
+        possible_create_objs = restored_create_objs
+
     def _query_missing_dict_model(self, field_name, fks, create_objs, status):
-        """Find missing dict type m2m models with a supplied filter method."""
-        count = 0
+        """Find missing dict type m2m models."""
         possible_objs = fks.pop(field_name, None)
         if not possible_objs:
-            return count
+            return 0
 
+        # Create possible_create_objs & a query filter map and cache the urls for
+        # identifiers to be created, but not queried against
         possible_create_objs = set()
         query_filter_map = {}
-        # create the filter
-
-        if field_name == "identifiers":
-            rel_map = IDENTIFIERS_MODEL_REL_MAP
-            query_missing_method = self._query_missing_identifiers_model_obj
-        else:
-            rel_map = DICT_MODEL_REL_MAP
-            query_missing_method = self._query_missing_dict_model_obj
-
-        model, key_rel, value_rel = rel_map[field_name]
-
-        for item in possible_objs.items():
-            value_filter, filter_and_prefix = query_missing_method(
-                key_rel, value_rel, item, possible_create_objs
-            )
-            self._add_value_filter_map(
-                value_filter, filter_and_prefix, query_filter_map
+        url_restore_map = {}
+        for key, values in possible_objs.items():
+            self._query_missing_dict_model_add_to_query_filter_map(
+                key,
+                values,
+                field_name,
+                possible_create_objs,
+                query_filter_map,
+                url_restore_map,
             )
 
-        # get the obj metadata
+        # Build combined query object from the value_filter
+        combined_q_obj = Q()
         for filter_and_prefix, value_filter in query_filter_map.items():
-            count += self._query_create_metadata(
-                model, possible_create_objs, value_filter, filter_and_prefix, status
-            )
+            combined_q_obj |= filter_and_prefix & value_filter
+
+        # Finally run the query and get only the correct create_objs
+        model = DICT_MODEL_FIELD_MODEL_MAP[field_name]
+        self._query_create_metadata(
+            model, possible_create_objs, None, combined_q_obj, status
+        )
+        self._query_missing_dict_model_identifiers_restore_urls(
+            field_name, possible_create_objs, url_restore_map
+        )
 
+        # Final Cleanup
         create_objs.update(possible_create_objs)
         count = len(create_objs)
         if count:
+            model = DICT_MODEL_FIELD_MODEL_MAP[field_name]
             vnp = model._meta.verbose_name_plural
             vnp = vnp.title() if vnp else "Nothings"
             self.log.info(f"Prepared {count} new {vnp}.")

codex/librarian/importer/update_comics.py

@@ -33,7 +33,6 @@ class UpdateComicsMixin(LinkComicsMixin):
                         value = default_value
                 setattr(comic, field_name, value)
             comic.presave()
-            comic.set_stat()
             comic.updated_at = now
             update_comics, comic_pks, comic_update_paths = results
             update_comics.append(comic)

codex/librarian/janitor/cleanup.py

@@ -2,26 +2,35 @@
 
 import logging
 from time import time
+from types import MappingProxyType
 
 from django.contrib.sessions.models import Session
 from django.utils.timezone import now
 
 from codex.librarian.janitor.status import JanitorStatusTypes
 from codex.models import (
+    AgeRating,
     Character,
     Contributor,
     ContributorPerson,
     ContributorRole,
+    Country,
     Folder,
     Genre,
+    Identifier,
+    IdentifierType,
     Imprint,
+    Language,
     Location,
+    OriginalFormat,
     Publisher,
+    ScanInfo,
     Series,
     SeriesGroup,
     StoryArc,
     StoryArcNumber,
     Tag,
+    Tagger,
     Team,
     Volume,
 )
@@ -29,33 +38,45 @@ from codex.status import Status
 from codex.worker_base import WorkerBaseMixin
 
 _COMIC_FK_CLASSES = (
-    Volume,
-    Series,
-    Imprint,
-    Publisher,
-    Folder,
+    AgeRating,
+    Country,
     Contributor,
-    Tag,
-    Team,
     Character,
+    Genre,
+    Folder,
+    Language,
     Location,
+    Identifier,
+    Imprint,
+    OriginalFormat,
+    Publisher,
+    Series,
     SeriesGroup,
+    ScanInfo,
     StoryArcNumber,
-    Genre,
+    Tagger,
+    Tag,
+    Team,
+    Volume,
 )
 _CONTRIBUTOR_FK_CLASSES = (ContributorRole, ContributorPerson)
 _STORY_ARC_NUMBER_FK_CLASSES = (StoryArc,)
+_IDENTIFIER_FK_CLASSES = (IdentifierType,)
 TOTAL_NUM_FK_CLASSES = (
     len(_COMIC_FK_CLASSES)
     + len(_CONTRIBUTOR_FK_CLASSES)
     + len(_STORY_ARC_NUMBER_FK_CLASSES)
+    + len(_IDENTIFIER_FK_CLASSES)
 )
 
-CLEANUP_MAP = [
-    (_COMIC_FK_CLASSES, "comic"),
-    (_CONTRIBUTOR_FK_CLASSES, "contributor"),
-    (_STORY_ARC_NUMBER_FK_CLASSES, "storyarcnumber"),
-]
+CLEANUP_MAP = MappingProxyType(
+    {
+        "comic": _COMIC_FK_CLASSES,
+        "contributor": _CONTRIBUTOR_FK_CLASSES,
+        "storyarcnumber": _STORY_ARC_NUMBER_FK_CLASSES,
+        "identifier": _IDENTIFIER_FK_CLASSES,
+    }
+)
 
 
 class CleanupMixin(WorkerBaseMixin):
@@ -79,7 +100,7 @@ class CleanupMixin(WorkerBaseMixin):
         try:
             self.status_controller.start(status)
             self.log.debug("Cleaning up unused foreign keys...")
-            for classes, field_name in CLEANUP_MAP:
+            for field_name, classes in CLEANUP_MAP.items():
                 self._bulk_cleanup_fks(classes, field_name, status)
             level = logging.INFO if status.complete else logging.DEBUG
             self.log.log(level, f"Cleaned up {status.complete} unused foreign keys.")

codex/migrations/0027_sort_name.py

@@ -0,0 +1,149 @@
+"""Generated by Django 5.0.4 on 2024-05-05 23:11."""
+
+import os
+from contextlib import suppress
+from pathlib import Path
+
+from django.db import migrations, models
+
+from codex.models.const import ARTICLES
+
+CONFIG_PATH = Path(os.environ.get("CODEX_CONFIG_DIR", Path.cwd() / "config"))
+COVER_ROOT = CONFIG_PATH / "cache" / "covers"
+_GROUP_MODEL_NAMES = (
+    "publisher",
+    "imprint",
+    "series",
+    "storyarc",
+    "folder",
+    "comic",
+    "failedimport",
+)
+
+
+def _set_sort_name(obj):
+    """Create sort_name for model."""
+    # Duplicate code from BaseGroupModel
+    sort_name = ""
+    lower_name = obj.name.lower()
+    sort_name = lower_name
+    name_parts = lower_name.split()
+    if len(name_parts) > 1:
+        first_word = name_parts[0]
+        if first_word in ARTICLES:
+            sort_name = " ".join(name_parts[1:])
+            sort_name += ", " + first_word
+    obj.sort_name = sort_name
+
+
+def _generate_sort_name(apps, _schema_editor):
+    """Update new sort_name field."""
+    for model_name in _GROUP_MODEL_NAMES:
+        model = apps.get_model("codex", model_name)
+        update_fields = ("sort_name",)
+        only_fields = ("name", *update_fields)
+        update_groups = []
+        objs = model.objects.only(*only_fields)
+        for obj in objs:
+            _set_sort_name(obj)
+            update_groups.append(obj)
+        if update_groups:
+            model.objects.bulk_update(update_groups, update_fields)
+            print(f"\tUpdated sort_name for {len(update_groups)} {model_name}s")
+
+
+def _remove_cover_symlinks(_apps, _schema_editor):
+    """Remove old missing cover symlinks."""
+    # Handled programmatically now.
+    count = 0
+    for dirpath_str, _, filenames in os.walk(COVER_ROOT):
+        dirpath = Path(dirpath_str)
+        for filename in filenames:
+            full_path = dirpath / filename
+            if full_path.is_symlink():
+                with suppress(OSError):
+                    full_path.unlink(missing_ok=True)
+                    count += 1
+    if count:
+        print(f"Removed {count} missing-cover symlinks from cover cache.")
+
+
+class Migration(migrations.Migration):
+    """Migrate."""
+
+    dependencies = [
+        ("codex", "0026_comicbox_1"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="comic",
+            name="sort_name",
+            field=models.CharField(db_index=True, default="", max_length=128),
+        ),
+        migrations.AddField(
+            model_name="failedimport",
+            name="sort_name",
+            field=models.CharField(db_index=True, default="", max_length=128),
+        ),
+        migrations.AddField(
+            model_name="folder",
+            name="sort_name",
+            field=models.CharField(db_index=True, default="", max_length=128),
+        ),
+        migrations.AddField(
+            model_name="imprint",
+            name="sort_name",
+            field=models.CharField(db_index=True, default="", max_length=128),
+        ),
+        migrations.AddField(
+            model_name="publisher",
+            name="sort_name",
+            field=models.CharField(db_index=True, default="", max_length=128),
+        ),
+        migrations.AddField(
+            model_name="series",
+            name="sort_name",
+            field=models.CharField(db_index=True, default="", max_length=128),
+        ),
+        migrations.AddField(
+            model_name="storyarc",
+            name="sort_name",
+            field=models.CharField(db_index=True, default="", max_length=128),
+        ),
+        migrations.AlterField(
+            model_name="adminflag",
+            name="key",
+            field=models.CharField(
+                choices=[
+                    ("FV", "Folder View"),
+                    ("RG", "Registration"),
+                    ("NU", "Non Users"),
+                    ("AU", "Auto Update"),
+                    ("SO", "Search Index Optimize"),
+                    ("IM", "Import Metadata"),
+                ],
+                db_index=True,
+                max_length=2,
+            ),
+        ),
+        migrations.AlterField(
+            model_name="adminflag",
+            name="key",
+            field=models.CharField(
+                choices=[
+                    ("FV", "Folder View"),
+                    ("RG", "Registration"),
+                    ("NU", "Non Users"),
+                    ("AU", "Auto Update"),
+                    ("SO", "Search Index Optimize"),
+                    ("IM", "Import Metadata"),
+                    ("DC", "Dynamic Group Covers"),
+                ],
+                db_index=True,
+                max_length=2,
+            ),
+        ),
+        migrations.RunPython(_generate_sort_name),
+        migrations.RunPython(_remove_cover_symlinks),
+    ]

codex/models/admin.py

@@ -39,6 +39,7 @@ class AdminFlag(BaseModel):
         AUTO_UPDATE = "AU"
         SEARCH_INDEX_OPTIMIZE = "SO"
         IMPORT_METADATA = "IM"
+        DYNAMIC_GROUP_COVERS = "DC"
 
     FALSE_DEFAULTS = frozenset({FlagChoices.AUTO_UPDATE})
 

codex/models/base.py

@@ -3,6 +3,8 @@
 from django.db.models import DateTimeField, Model
 from django.db.models.base import ModelBase
 
+from codex.models.query import GroupByManager
+
 __all__ = ("BaseModel",)
 
 MAX_NAME_LEN = 128
@@ -14,6 +16,7 @@ class BaseModel(Model):
 
     created_at = DateTimeField(auto_now_add=True)
     updated_at = DateTimeField(auto_now=True)
+    objects = GroupByManager()
 
     class Meta(ModelBase):  # type: ignore
         """Without this a real table is created and joined to."""