codex 1.6.0__py3-none-any.whl → 1.6.0a1__py3-none-any.whl

codex/librarian/importer/{aggregate.py → aggregate_metadata.py}

@@ -2,7 +2,13 @@
 
 from collections.abc import Mapping
 from pathlib import Path
+from zipfile import BadZipFile
 
+from comicbox.box import Comicbox
+from comicbox.exceptions import UnsupportedArchiveTypeError
+from rarfile import BadRarFile
+
+from codex.librarian.importer.clean_metadata import CleanMetadataMixin
 from codex.librarian.importer.const import (
     COMIC_FK_FIELD_NAMES,
     COMIC_M2M_FIELD_NAMES,
@@ -18,15 +24,13 @@ from codex.librarian.importer.const import (
     MDS,
     VOLUME_COUNT,
 )
-from codex.librarian.importer.extract import ExtractMetadataImporter
-from codex.librarian.importer.status import ImportStatusTypes
+from codex.librarian.importer.status import ImportStatusTypes, status_notify
 from codex.models import Imprint, Publisher, Series, Volume
 from codex.models.admin import AdminFlag
 from codex.status import Status
-from codex.util import max_none
 
 
-class AggregateMetadataImporter(ExtractMetadataImporter):
+class AggregateMetadataMixin(CleanMetadataMixin):
     """Aggregate metadata from comics to prepare for importing."""
 
     _BROWSER_GROUPS = (Publisher, Imprint, Series, Volume)
@@ -82,13 +86,37 @@ class AggregateMetadataImporter(ExtractMetadataImporter):
         m2m_md[FOLDERS_FIELD] = Path(path).parents
         return m2m_md
 
-    def _get_path_metadata(self, md, path):
-        """Get the metadata from comicbox and munge it a little."""
-        group_tree_md = self._get_group_tree(md)
-        fk_md = self._get_fk_metadata(md)
-        m2m_md = self._get_m2m_metadata(md, path)
-
-        return md, m2m_md, fk_md, group_tree_md
+    def _get_path_metadata(self, path, import_metadata):
+        """Get the metatada from comicbox and munge it a little."""
+        md = {}
+        fk_md = {}
+        group_tree_md = {}
+        m2m_md = {}
+        failed_import = {}
+        try:
+            if import_metadata:
+                with Comicbox(path) as cb:
+                    md = cb.to_dict()
+                    md = md.get("comicbox", {})
+                    if "file_type" not in md:
+                        md["file_type"] = cb.get_file_type()
+                    if "page_count" not in md:
+                        md["page_count"] = cb.get_page_count()
+
+            md["path"] = path
+            md = self.clean_md(md)
+
+            group_tree_md = self._get_group_tree(md)
+            fk_md = self._get_fk_metadata(md)
+            m2m_md = self._get_m2m_metadata(md, path)
+
+        except (UnsupportedArchiveTypeError, BadRarFile, BadZipFile, OSError) as exc:
+            self.log.warning(f"Failed to import {path}: {exc}")
+            failed_import = {path: exc}
+        except Exception as exc:
+            self.log.exception(f"Failed to import: {path}")
+            failed_import = {path: exc}
+        return md, m2m_md, fk_md, group_tree_md, failed_import
 
     @staticmethod
     def _aggregate_m2m_metadata_dict_value(names, key, values, all_fks):
@@ -154,13 +182,22 @@ class AggregateMetadataImporter(ExtractMetadataImporter):
                 all_fks[field] = set()
             all_fks[field].add(name)
 
+    @staticmethod
+    def _none_max(a, b):
+        """None aware math.max."""
+        if a is not None and b is not None:
+            return max(a, b)
+        if a is None:
+            return b
+        return a
+
     @classmethod
     def _set_max_group_count(cls, common_args, group_class, index, count_key):
         """Assign the maximum group count number."""
         all_fks, group_tree, group_md = common_args
         group_name = group_tree[0:index]
         try:
-            count = max_none(
+            count = cls._none_max(
                 all_fks[GROUP_TREES][Series].get(group_name),
                 group_md.get(count_key),
             )
@@ -178,79 +215,72 @@ class AggregateMetadataImporter(ExtractMetadataImporter):
             cls._set_max_group_count(common_args, Series, 3, VOLUME_COUNT)
             cls._set_max_group_count(common_args, Volume, 4, ISSUE_COUNT)
 
-    def _aggregate_path(self, md, path, status):
+    def _aggregate_path(self, data, path, import_metadata):
         """Aggregate metadata for one path."""
-        md, m2m_md, fk_md, group_tree_md = self._get_path_metadata(md, path)
-
         path_str = str(path)
-        if md:
-            all_mds = self.metadata[MDS]
-            all_mds[path_str] = md
+        md, m2m_md, fk_md, group_tree_md, failed_import = self._get_path_metadata(
+            path_str, import_metadata
+        )
 
-        all_fks = self.metadata[FKS]
-        if m2m_md:
-            all_m2m_mds = self.metadata[M2M_MDS]
-            self._aggregate_m2m_metadata(all_m2m_mds, m2m_md, all_fks, path_str)
+        all_failed_imports, all_mds, all_m2m_mds, all_fks, status = data
+        if failed_import:
+            all_failed_imports.update(failed_import)
+        else:
+            if md:
+                all_mds[path_str] = md
+
+            if m2m_md:
+                self._aggregate_m2m_metadata(all_m2m_mds, m2m_md, all_fks, path_str)
 
-        if fk_md:
-            self._aggregate_fk_metadata(all_fks, fk_md)
+            if fk_md:
+                self._aggregate_fk_metadata(all_fks, fk_md)
 
-        if group_tree_md:
-            self._aggregate_group_tree_metadata(all_fks, group_tree_md)
+            if group_tree_md:
+                self._aggregate_group_tree_metadata(all_fks, group_tree_md)
 
         if status:
             status.complete += 1
             self.status_controller.update(status)
 
-    def get_aggregate_metadata(
+    @status_notify(status_type=ImportStatusTypes.AGGREGATE_TAGS)
+    def get_aggregate_metadata(  # noqa: PLR0913
         self,
+        all_paths,
+        library_path,
+        metadata,
+        force_import_metadata,
         status=None,
     ):
-        """Get aggregated metadata for the paths given."""
-        all_paths = self.task.files_modified | self.task.files_created
+        """Get aggregated metatada for the paths given."""
         total_paths = len(all_paths)
-
         if not total_paths:
             return 0
-        self.log.info(
-            f"Reading tags from {total_paths} comics in {self.library.path}..."
-        )
-        status = Status(ImportStatusTypes.AGGREGATE_TAGS, 0, total_paths)
-        self.status_controller.start(status, notify=False)
-
-        # Set import_metadata flag
-        if self.task.force_import_metadata:
+        self.log.info(f"Reading tags from {total_paths} comics in {library_path}...")
+        all_mds = metadata[MDS]
+        all_m2m_mds = metadata[M2M_MDS]
+        all_fks = metadata[FKS]
+        all_fks[GROUP_TREES] = {cls: {} for cls in self._BROWSER_GROUPS}
+        all_failed_imports = metadata[FIS]
+
+        if status and status.complete is None:
+            status.complete = 0
+        key = AdminFlag.FlagChoices.IMPORT_METADATA.value  # type: ignore
+        if force_import_metadata:
             import_metadata = True
         else:
-            key = AdminFlag.FlagChoices.IMPORT_METADATA.value  # type: ignore
             import_metadata = AdminFlag.objects.get(key=key).on
         if not import_metadata:
             self.log.warn("Admin flag set to NOT import metadata.")
-
-        # Init metadata, extract and aggregate
-        self.metadata[MDS] = {}
-        self.metadata[M2M_MDS] = {}
-        self.metadata[FKS] = {GROUP_TREES: {cls: {} for cls in self._BROWSER_GROUPS}}
-        self.metadata[FIS] = {}
+        data = (all_failed_imports, all_mds, all_m2m_mds, all_fks, status)
         for path in all_paths:
-            md = self.extract_and_clean(path, import_metadata)
-            if md:
-                self._aggregate_path(md, path, status)
+            self._aggregate_path(data, path, import_metadata)
 
-        # Aggregate further
-        self.metadata[FKS][COMIC_PATHS] = frozenset(self.metadata[MDS].keys())
-        fis = self.metadata[FIS]
-        self.task.files_modified -= fis.keys()
-        self.task.files_created -= fis.keys()
-
-        # Set statii
-        fi_status = Status(ImportStatusTypes.FAILED_IMPORTS, 0, len(fis))
+        all_fks[COMIC_PATHS] = frozenset(all_mds.keys())
+        fi_status = Status(ImportStatusTypes.FAILED_IMPORTS, 0, len(all_failed_imports))
         self.status_controller.update(
             fi_status,
             notify=False,
         )
         count = status.complete if status else 0
         self.log.info(f"Aggregated tags from {count} comics.")
-
-        self.status_controller.finish(status)
         return count

codex/librarian/importer/{extract.py → clean_metadata.py}

@@ -3,18 +3,13 @@
 from contextlib import suppress
 from decimal import ROUND_DOWN, Decimal
 from typing import Any
-from zipfile import BadZipFile
 
-from comicbox.box import Comicbox
 from comicbox.box.computed import IDENTIFIERS_KEY
-from comicbox.exceptions import UnsupportedArchiveTypeError
 from comicbox.schemas.comicbox_mixin import CONTRIBUTORS_KEY
 from django.db.models import CharField
 from django.db.models.fields import DecimalField, PositiveSmallIntegerField
-from rarfile import BadRarFile
 
-from codex.librarian.importer.const import FIS, STORY_ARCS_METADATA_KEY
-from codex.librarian.importer.query_fks import QueryForeignKeysImporter
+from codex.librarian.importer.const import STORY_ARCS_METADATA_KEY
 from codex.models import Comic
 from codex.models.named import (
     ContributorPerson,
@@ -23,6 +18,7 @@ from codex.models.named import (
     IdentifierType,
     StoryArc,
 )
+from codex.threads import QueuedThread
 
 _MD_INVALID_KEYS = frozenset(
     (
@@ -66,7 +62,7 @@ _DECIMAL_ZERO = Decimal("0.00")
 _ALPHA_2_LEN = 2
 
 
-class ExtractMetadataImporter(QueryForeignKeysImporter):
+class CleanMetadataMixin(QueuedThread):
     """Clean metadata before importing."""
 
     @staticmethod
@@ -223,7 +219,7 @@ class ExtractMetadataImporter(QueryForeignKeysImporter):
         cls._assign_or_pop(md, IDENTIFIERS_KEY, clean_identifiers)
 
     @classmethod
-    def _clean_md(cls, md):
+    def clean_md(cls, md):
         """Sanitize the metadata before import."""
         cls._title_to_name(md)
         md = cls._prune_extra_keys(md)
@@ -235,28 +231,3 @@ class ExtractMetadataImporter(QueryForeignKeysImporter):
         cls._clean_story_arcs(md)
         cls._clean_identifiers(md)
         return md
-
-    def extract_and_clean(self, path, import_metadata):
-        """Extract metadata from comic and clean it for codex."""
-        md = {}
-        failed_import = {}
-        try:
-            if import_metadata:
-                with Comicbox(path) as cb:
-                    md = cb.to_dict()
-                    md = md.get("comicbox", {})
-                    if "file_type" not in md:
-                        md["file_type"] = cb.get_file_type()
-                    if "page_count" not in md:
-                        md["page_count"] = cb.get_page_count()
-            md["path"] = path
-            md = self._clean_md(md)
-        except (UnsupportedArchiveTypeError, BadRarFile, BadZipFile, OSError) as exc:
-            self.log.warning(f"Failed to import {path}: {exc}")
-            failed_import = {path: exc}
-        except Exception as exc:
-            self.log.exception(f"Failed to import: {path}")
-            failed_import = {path: exc}
-        if failed_import:
-            self.metadata[FIS].update(failed_import)
-        return md

codex/librarian/importer/const.py

@@ -2,22 +2,17 @@
 
 from types import MappingProxyType
 
-from bidict import bidict
-
 from codex.models import (
     Comic,
     Contributor,
-    Folder,
     Imprint,
     Publisher,
     Series,
-    StoryArc,
     StoryArcNumber,
     Volume,
 )
 from codex.models.groups import BrowserGroupModel
 from codex.models.named import Identifier
-from codex.models.paths import CustomCover
 
 #################
 # DICT METADATA #
@@ -71,6 +66,10 @@ DICT_MODEL_REL_MAP = MappingProxyType(
             f"{_STORY_ARC_FIELD_NAME}__name",
             _NUMBER_FIELD_NAME,
         ),
+    }
+)
+IDENTIFIERS_MODEL_REL_MAP = MappingProxyType(
+    {
         IDENTIFIERS_FIELD_NAME: (
             f"{_IDENTIFIER_TYPE_FIELD_NAME}__name",
             {
@@ -115,7 +114,12 @@ DICT_MODEL_REL_LINK_MAP = MappingProxyType(
 # BULK UPDATE #
 ###############
 
-_EXCLUDEBULK_UPDATE_COMIC_FIELDS = {"bookmark", "created_at", "id", "library"}
+_EXCLUDEBULK_UPDATE_COMIC_FIELDS = {
+    "created_at",
+    "searchresult",
+    "id",
+    "bookmark",
+}
 GROUP_BASE_FIELDS = ("name", "sort_name")
 BULK_UPDATE_COMIC_FIELDS = tuple(
     sorted(
@@ -125,7 +129,6 @@ BULK_UPDATE_COMIC_FIELDS = tuple(
         and (field.name not in _EXCLUDEBULK_UPDATE_COMIC_FIELDS)
     )
 )
-BULK_CREATE_COMIC_FIELDS = (*BULK_UPDATE_COMIC_FIELDS, "library")
 BULK_UPDATE_FOLDER_FIELDS = (
     *GROUP_BASE_FIELDS,
     "stat",
@@ -140,15 +143,9 @@ BULK_UPDATE_COMIC_FIELDS_WITH_VALUES = tuple(
         - frozenset(BULK_UPDATE_FOLDER_MODIFIED_FIELDS)
     )
 )
-MOVED_BULK_COMIC_UPDATE_FIELDS = ("path", "parent_folder", "stat", "updated_at")
-MOVED_BULK_FOLDER_UPDATE_FIELDS = (
-    "path",
-    "parent_folder",
-    *GROUP_BASE_FIELDS,
-    "stat",
-    "updated_at",
-)
-CUSTOM_COVER_UPDATE_FIELDS = ("path", "stat", "updated_at", "sort_name", "group")
+MOVED_BULK_COMIC_UPDATE_FIELDS = ("path", "parent_folder")
+MOVED_BULK_FOLDER_UPDATE_FIELDS = ("path", "parent_folder", *GROUP_BASE_FIELDS)
+
 
 #########
 # OTHER #
@@ -185,25 +182,12 @@ MDS = "mds"
 M2M_MDS = "m2m_mds"
 FKS = "fks"
 FIS = "fis"
-FK_CREATE = "fk_create"
-COVERS_UPDATE = "covers_update"
-COVERS_CREATE = "covers_create"
-LINK_COVER_PKS = "link_cover_pks"
 GROUP_COMPARE_FIELDS = MappingProxyType(
     {
         Series: ("publisher__name", "imprint__name", "name"),
         Volume: ("publisher__name", "imprint__name", "series__name", "name"),
     }
 )
-CLASS_CUSTOM_COVER_GROUP_MAP = bidict(
-    {
-        Publisher: CustomCover.GroupChoice.P.value,
-        Imprint: CustomCover.GroupChoice.I.value,
-        Series: CustomCover.GroupChoice.S.value,
-        StoryArc: CustomCover.GroupChoice.A.value,
-        Folder: CustomCover.GroupChoice.F.value,
-    }
-)
 
 COMIC_FK_FIELD_NAMES = frozenset(
     field.name
@@ -219,20 +203,3 @@ COMIC_M2M_FIELD_NAMES = frozenset(
     for field in Comic._meta.get_fields()
     if field.many_to_many and field.name != "folders"
 )
-COMIC_GROUP_FIELD_NAMES = (
-    "publisher",
-    "imprint",
-    "series",
-    "volume",
-    "story_arc_numbers",
-    "folders",
-)
-
-FKC_CONTRIBUTORS = "create_contributors"
-FKC_STORY_ARC_NUMBERS = "create_story_arc_numbers"
-FKC_IDENTIFIERS = "create_identifiers"
-FKC_CREATE_GROUPS = "create_groups"
-FKC_UPDATE_GROUPS = "update_groups"
-FKC_CREATE_FKS = "create_fks"
-FKC_FOLDER_PATHS = "create_folder_paths"
-FKC_TOTAL_FKS = "total_fks"

codex/librarian/importer/create_comics.py

@@ -1,121 +1,39 @@
 """Bulk update and create comic objects and bulk update m2m fields."""
 
-from django.db.models import NOT_PROVIDED
-from django.db.models.functions import Now
-
-from codex.librarian.importer.const import (
-    BULK_CREATE_COMIC_FIELDS,
-    BULK_UPDATE_COMIC_FIELDS,
-    BULK_UPDATE_COMIC_FIELDS_WITH_VALUES,
-    MDS,
-)
-from codex.librarian.importer.link_comics import LinkComicsImporter
-from codex.librarian.importer.status import ImportStatusTypes
+from codex.librarian.importer.const import BULK_UPDATE_COMIC_FIELDS
+from codex.librarian.importer.link_comics import LinkComicsMixin
+from codex.librarian.importer.status import ImportStatusTypes, status_notify
 from codex.models import (
     Comic,
 )
-from codex.status import Status
 
 
-class CreateComicsImporter(LinkComicsImporter):
+class CreateComicsMixin(LinkComicsMixin):
     """Create comics methods."""
 
-    def _bulk_update_comics_add_comic(self, comic, results):
-        """Add one comic and stats to the bulk update list."""
-        try:
-            md = self.metadata[MDS].pop(comic.path)
-            self.get_comic_fk_links(md, comic.path)
-            for field_name in BULK_UPDATE_COMIC_FIELDS_WITH_VALUES:
-                value = md.get(field_name)
-                if value is None:
-                    default_value = Comic._meta.get_field(field_name).default
-                    if default_value != NOT_PROVIDED:
-                        value = default_value
-                setattr(comic, field_name, value)
-            comic.presave()
-            comic.updated_at = Now()
-            update_comics, comic_pks, comic_update_paths = results
-            update_comics.append(comic)
-            comic_pks.append(comic.pk)
-            comic_update_paths.add(comic.path)
-        except Exception:
-            self.log.exception(f"Error preparing {comic} for update.")
-
-    def bulk_update_comics(self):
-        """Bulk update comics, and move nonextant comics into create job.."""
-        num_comics = len(self.task.files_modified)
-        if not num_comics:
-            return num_comics
-
-        self.log.debug(
-            f"Preparing {num_comics} comics for update in library {self.library.path}."
-        )
-        status = Status(ImportStatusTypes.FILES_MODIFIED, 0, num_comics)
-        self.status_controller.start(status, notify=False)
-        # Get existing comics to update
-        comics = Comic.objects.filter(
-            library=self.library, path__in=self.task.files_modified
-        ).only(*BULK_UPDATE_COMIC_FIELDS)
-
-        # set attributes for each comic
-        update_comics = []
-        comic_pks = []
-        comic_update_paths = set()
-        results = update_comics, comic_pks, comic_update_paths
-        for comic in comics.iterator():
-            self._bulk_update_comics_add_comic(comic, results)
-
-        converted_create_paths = frozenset(
-            set(self.task.files_modified) - comic_update_paths
-        )
-        self.task.files_created |= converted_create_paths
-        count = len(converted_create_paths)
-        if count:
-            self.log.info(f"Converted {count} update paths to create paths.")
-
-        self.log.debug(f"Bulk updating {len(update_comics)} comics.")
-        try:
-            Comic.objects.bulk_update(update_comics, BULK_UPDATE_COMIC_FIELDS)
-            count = len(update_comics)
-
-            self._remove_covers(comic_pks, False)  # type: ignore
-            self.log.debug(f"Purging covers for {len(comic_pks)} updated comics.")
-            if count:
-                self.log.info(f"Updated {count} comics.")
-        except Exception:
-            self.log.exception(f"While updating {comic_update_paths}")
-
-        self.task.files_modified = frozenset()
-
-        self.status_controller.finish(status)
-        return count
-
-    def bulk_create_comics(self):
+    @status_notify(status_type=ImportStatusTypes.FILES_CREATED, updates=False)
+    def bulk_create_comics(self, comic_paths, library, mds, **kwargs):
         """Bulk create comics."""
-        num_comics = len(self.task.files_created)
-        if not num_comics:
-            return num_comics
+        if not comic_paths:
+            return 0
+        num_comics = len(comic_paths)
+
         # prepare create comics
         self.log.debug(
-            f"Preparing {num_comics} comics for creation in library {self.library.path}."
+            f"Preparing {num_comics} comics for creation in library {library.path}."
         )
-        status = Status(ImportStatusTypes.FILES_CREATED, 0, num_comics)
-        self.status_controller.start(status, notify=False)
-
         create_comics = []
-        for path in sorted(self.task.files_created):
+        for path in comic_paths:
             try:
-                md = self.metadata[MDS].pop(path, {})
-                self.get_comic_fk_links(md, path)
-                comic = Comic(**md, library=self.library)
+                md = mds.pop(path, {})
+                self.get_comic_fk_links(md, library, path)
+                comic = Comic(**md)
                 comic.presave()
                 create_comics.append(comic)
             except KeyError:
                 self.log.warning(f"No comic metadata for {path}")
             except Exception:
                 self.log.exception(f"Error preparing {path} for create.")
-        self.task.files_created = frozenset()
-        self.metadata.pop(MDS)
 
         num_comics = len(create_comics)
         count = 0
@@ -125,15 +43,13 @@ class CreateComicsImporter(LinkComicsImporter):
                 Comic.objects.bulk_create(
                     create_comics,
                     update_conflicts=True,
-                    update_fields=BULK_CREATE_COMIC_FIELDS,
+                    update_fields=BULK_UPDATE_COMIC_FIELDS,
                     unique_fields=Comic._meta.unique_together[0],  # type: ignore
                 )
                 count = len(create_comics)
                 if count:
                     self.log.info(f"Created {count} comics.")
             except Exception:
-                self.log.exception(f"While creating {num_comics} comics")
+                self.log.exception(f"While creating {comic_paths}")
 
-        self.changed += count
-        self.status_controller.finish(status)
         return count