codex 1.4.0a0__py3-none-any.whl → 1.4.1__py3-none-any.whl

codex/config_default.yaml

@@ -1,12 +1,20 @@
 comicbox:
-  comet: True
-  comicbookinfo: True
-  comicinfoxml: True
   delete_orig: False
   delete_tags: False
   dest_path: .
   dry_run: False
-  filename: True
   raw: False
+  read_comet: True
+  read_comicbookinfo: True
+  read_comicinfoxml: True
+  read_filename: True
+  read_pdf: True
   recurse: False
   metadata: {}
+  close_fd: True
+  check_unrar: False
+  write_comet: False
+  write_comicbookinfo: False
+  write_comicinfoxml: False
+  write_filename: False
+  write_pdf: False

codex/db_functions.py

@@ -3,9 +3,11 @@ from django.db.models import Aggregate
 from django.db.models.fields import CharField
 
 
-# TODO unused
 class GroupConcat(Aggregate):
-    """Sqlite3 group_concat() function."""
+    """Sqlite3 group_concat() function.
+
+    Unused.
+    """
 
     function = "GROUP_CONCAT"
     template = "%(function)s(%(distinct)s %(expressions)s)"

codex/integrity.py

@@ -31,17 +31,19 @@ MIGRATION_0010 = "0010_haystack"
 MIGRATION_0011 = "0010_library_groups_and_metadata_changes"
 MIGRATION_0018 = "0018_rename_userbookmark_bookmark"
 MIGRATION_0023 = "0023_rename_credit_creator_and_more"
+MIGRATION_0025 = "0025_add_story_arc_number"
 M2M_NAMES = {
     "Character": "characters",
     "Creator": "creators",
     "Genre": "genres",
     "Location": "locations",
     "SeriesGroup": "series_groups",
-    "StoryArc": "story_arcs",
+    "StoryArcNumber": "story_arc_numbers",
     "Tag": "tags",
     "Team": "teams",
     "Folder": "folders",
 }
+SKIP_M2M_CHECKS = {"Creator": MIGRATION_0023, "StoryArcNumber": MIGRATION_0025}
 NULL_SET = frozenset([None])
 HAVE_LIBRARY_FKS = ("FailedImport", "Folder", "Comic")
 GROUP_HOSTS = {
@@ -68,6 +70,7 @@ DELETE_BAD_COMIC_FOLDER_RELATIONS_SQL = (
     'IN (SELECT "codex_folder"."id" FROM "codex_folder")))'
 )
 MIGRATION_DIR = CODEX_PATH / "migrations"
+
 LOG = get_logger(__name__)
 
 
@@ -289,16 +292,24 @@ def _delete_errors():
     _delete_old_comic_folder_fks()
 
 
+def _check_field_for_migration(model_name):
+    """Skip some models before migrations."""
+    for check_model_name, migration in SKIP_M2M_CHECKS.items():
+        if model_name == check_model_name and not has_applied_migration(migration):
+            LOG.debug(
+                f"Skipping {check_model_name} integrity check until"
+                f"migration {migration} applied."
+            )
+            return True
+    return False
+
+
 def _repair_integrity():
     """REPAIR the objects that are left."""
     comic_model = apps.get_model("codex", "Comic")
     bad_comic_ids = comic_model.objects.none()
     for m2m_model_name, field_name in M2M_NAMES.items():
-        if m2m_model_name == "Creator" and not has_applied_migration(MIGRATION_0023):
-            LOG.debug(
-                "Skipping Creator integrity check until"
-                f"migration {MIGRATION_0023} applied."
-            )
+        if _check_field_for_migration(m2m_model_name):
             continue
         try:
             bad_comic_ids |= _fix_comic_m2m_integrity_errors(

codex/librarian/covers/coverd.py

@@ -2,8 +2,7 @@
 from codex.librarian.covers.create import CoverCreateMixin
 from codex.librarian.covers.purge import CoverPurgeMixin
 from codex.librarian.covers.tasks import (
-    CoverBulkCreateTask,
-    CoverCreateTask,
+    CoverCreateAllTask,
     CoverRemoveAllTask,
     CoverRemoveOrphansTask,
     CoverRemoveTask,
@@ -24,14 +23,7 @@ class CoverCreatorThread(CoverCreateMixin, CoverPurgeMixin):
             self.purge_comic_covers(task.comic_pks)
         elif isinstance(task, CoverRemoveOrphansTask):
             self.cleanup_orphan_covers()
-
-        #####################
-        # UNUSED BELOW HERE #
-        #####################
-
-        elif isinstance(task, CoverCreateTask):
-            self.create_cover(task.pk)
-        elif isinstance(task, CoverBulkCreateTask):
-            self.bulk_create_comic_covers(task.comic_pks)
+        elif isinstance(task, CoverCreateAllTask):
+            self.create_all_covers()
         else:
             self.log.error(f"Bad task sent to {self.__class__.__name__}: {task}")

codex/librarian/covers/create.py

@@ -10,7 +10,6 @@ from codex.librarian.covers.path import CoverPathMixin
 from codex.librarian.covers.status import CoverStatusTypes
 from codex.librarian.covers.tasks import CoverSaveToCache
 from codex.models import Comic
-from codex.pdf import PDF
 from codex.status import Status
 from codex.version import COMICBOX_CONFIG
 
@@ -39,13 +38,12 @@ class CoverCreateMixin(CoverPathMixin):
             return cover_thumb_buffer.getvalue()
 
     @classmethod
-    def _get_comic_cover_image(cls, comic):
+    def _get_comic_cover_image(cls, comic_path):
         """Create comic cover if none exists.
 
         Return image thumb data or path to missing file thumb.
         """
-        car_class = PDF if comic.file_type == Comic.FileType.PDF.value else ComicArchive
-        with car_class(comic.path, config=COMICBOX_CONFIG) as car:
+        with ComicArchive(comic_path, config=COMICBOX_CONFIG) as car:
             image_data = car.get_cover_image()
         if not image_data:
             reason = "Read empty cover"
@@ -58,14 +56,14 @@ class CoverCreateMixin(CoverPathMixin):
 
         Called from views/cover.
         """
-        comic = None
+        comic_path = None
         try:
-            comic = Comic.objects.only("path", "file_type").get(pk=pk)
-            cover_image = cls._get_comic_cover_image(comic)
+            comic_path = Comic.objects.only("path").get(pk=pk).path
+            cover_image = cls._get_comic_cover_image(comic_path)
             data = cls._create_cover_thumbnail(cover_image)
         except Exception as exc:
             data = b""
-            comic_str = comic.path if comic else f"{pk=}"
+            comic_str = comic_path if comic_path else f"{pk=}"
             log.warning(f"Could not create cover thumbnail for {comic_str}: {exc}")
 
         task = CoverSaveToCache(cover_path, data)
@@ -81,19 +79,8 @@ class CoverCreateMixin(CoverPathMixin):
         else:
             cover_path.symlink_to(self.MISSING_COVER_PATH)
 
-    #####################
-    # UNUSED BELOW HERE #
-    #####################
-
-    def create_cover(self, pk):
-        """Create a cover from a comic id."""
-        # XXX Unused.
-        cover_path = self.get_cover_path(pk)
-        self.create_cover_from_path(pk, cover_path, self.log, self.librarian_queue)
-
-    def bulk_create_comic_covers(self, comic_pks):
+    def _bulk_create_comic_covers(self, comic_pks):
         """Create bulk comic covers."""
-        # XXX Unused
         num_comics = len(comic_pks)
         if not num_comics:
             return None
@@ -111,7 +98,9 @@ class CoverCreateMixin(CoverPathMixin):
                     status.decrement_total()
                 else:
                     # bulk creator creates covers inline
-                    self.create_cover(pk)
+                    self.create_cover_from_path(
+                        pk, cover_path, self.log, self.librarian_queue
+                    )
                     status.increment_complete()
                 self.status_controller.update(status)
 
@@ -120,3 +109,8 @@ class CoverCreateMixin(CoverPathMixin):
         finally:
             self.status_controller.finish(status)
         return status.complete
+
+    def create_all_covers(self):
+        """Create all covers for all libraries."""
+        pks = Comic.objects.values_list("pk", flat=True)
+        self._bulk_create_comic_covers(pks)

codex/librarian/covers/tasks.py

@@ -39,20 +39,6 @@ class CoverSaveToCache(CoverTask):
     data: bytes
 
 
-#####################
-# UNUSED BELOW HERE #
-#####################
-
-
-@dataclass
-class CoverCreateTask(CoverTask):
-    """Create one comic cover."""
-
-    pk: int
-
-
 @dataclass
-class CoverBulkCreateTask(CoverTask):
-    """A list of comic src and dest paths."""
-
-    comic_pks: frozenset
+class CoverCreateAllTask(CoverTask):
+    """A create all comic covers."""

codex/librarian/importer/aggregate_metadata.py

@@ -4,13 +4,13 @@ from zipfile import BadZipFile
 
 from comicbox.comic_archive import ComicArchive
 from comicbox.exceptions import UnsupportedArchiveTypeError
+from confuse import AttrDict
 from rarfile import BadRarFile
 
 from codex.comic_field_names import COMIC_M2M_FIELD_NAMES
 from codex.librarian.importer.clean_metadata import CleanMetadataMixin
 from codex.librarian.importer.status import ImportStatusTypes, status_notify
 from codex.models import Comic, Imprint, Publisher, Series, Volume
-from codex.pdf import PDF
 from codex.status import Status
 from codex.version import COMICBOX_CONFIG
 
@@ -20,6 +20,10 @@ class AggregateMetadataMixin(CleanMetadataMixin):
 
     _BROWSER_GROUPS = (Publisher, Imprint, Series, Volume)
     _BROWSER_GROUP_TREE_COUNT_FIELDS = frozenset(["volume_count", "issue_count"])
+    _GROUP_TREES_INIT = {
+        "group_trees": {Publisher: {}, Imprint: {}, Series: {}, Volume: {}},
+    }
+    _AGGREGATE_COMICBOX_CONFIG = AttrDict({**COMICBOX_CONFIG, "close_fd": False})
 
     @staticmethod
     def _get_file_type(path):
@@ -39,8 +43,7 @@ class AggregateMetadataMixin(CleanMetadataMixin):
         group_tree_md = {}
         failed_import = {}
         try:
-            car_class = PDF if PDF.is_pdf(path) else ComicArchive
-            with car_class(path, config=COMICBOX_CONFIG, closefd=False) as car:
+            with ComicArchive(path, config=self._AGGREGATE_COMICBOX_CONFIG) as car:
                 md = car.get_metadata()
                 md["file_type"] = car.get_file_type()
 
@@ -79,27 +82,53 @@ class AggregateMetadataMixin(CleanMetadataMixin):
         return md, m2m_md, group_tree_md, failed_import
 
     @staticmethod
-    def _aggregate_m2m_metadata(all_m2m_mds, m2m_md, all_fks, path):
+    def _aggregate_m2m_metadata_creators(creator_dict_list, all_fks):
+        """Aggregate creators metadata."""
+        if not creator_dict_list:
+            return
+
+        if "creators" not in all_fks:
+            all_fks["creators"] = set()
+
+        for creator_dict in creator_dict_list:
+            # add the fk relations to fks to query.
+            for creator_field, name in creator_dict.items():
+                # These fields are ambiguous because they're fks to creator
+                #   but aren't ever in Comic so query_fks.py can
+                #   disambiguate with special code
+                if creator_field not in all_fks:
+                    all_fks[creator_field] = set()
+                all_fks[creator_field].add(name)
+
+            # Add creators to the all_fks list as well.
+            creator_tuple = tuple(sorted(creator_dict.items()))
+            all_fks["creators"].add(creator_tuple)
+
+    @staticmethod
+    def _aggregate_m2m_metadata_story_arc_numbers(story_arc_numbers_dict, all_fks):
+        """Aggregate story arc numbers."""
+        if not story_arc_numbers_dict:
+            return
+
+        if "story_arc" not in all_fks:
+            all_fks["story_arc"] = set()
+        all_fks["story_arc"] |= frozenset(story_arc_numbers_dict.keys())
+
+        if "story_arc_numbers" not in all_fks:
+            all_fks["story_arc_numbers"] = set()
+        all_fks["story_arc_numbers"] |= frozenset(story_arc_numbers_dict.items())
+
+    @classmethod
+    def _aggregate_m2m_metadata(cls, all_m2m_mds, m2m_md, all_fks, path):
         """Aggregate many to many metadata by ."""
         # m2m fields and fks
         all_m2m_mds[path] = m2m_md
         # aggregate fks
         for field, names in m2m_md.items():
             if field == "creators":
-                if names and "creators" not in all_fks:
-                    all_fks["creators"] = set()
-
-                # for creators add the creator fks to fks
-                for creator_dict in names:
-                    for creator_field, name in creator_dict.items():
-                        # These fields are ambiguous because they're fks to creator
-                        #   but aren't ever in Comic so query_fks.py can
-                        #   disambiguate with special code
-                        if creator_field not in all_fks:
-                            all_fks[creator_field] = set()
-                        all_fks[creator_field].add(name)
-                    creator_tuple = tuple(sorted(creator_dict.items()))
-                    all_fks["creators"].add(creator_tuple)
+                cls._aggregate_m2m_metadata_creators(names, all_fks)
+            elif field == "story_arc_numbers":
+                cls._aggregate_m2m_metadata_story_arc_numbers(names, all_fks)
             elif field != "folders":
                 if field not in all_fks:
                     all_fks[field] = set()
@@ -138,6 +167,28 @@ class AggregateMetadataMixin(CleanMetadataMixin):
             cls._set_max_group_count(common_args, Series, 3, "volume_count")
             cls._set_max_group_count(common_args, Volume, 4, "issue_count")
 
+    def _aggregate_path(self, data, path):
+        """Aggregate metadata for one path."""
+        path_str = str(path)
+        md, m2m_md, group_tree_md, failed_import = self._get_path_metadata(path_str)
+
+        all_failed_imports, all_mds, all_m2m_mds, all_fks, status = data
+        if failed_import:
+            all_failed_imports.update(failed_import)
+        else:
+            if md:
+                all_mds[path_str] = md
+
+            if m2m_md:
+                self._aggregate_m2m_metadata(all_m2m_mds, m2m_md, all_fks, path_str)
+
+            if group_tree_md:
+                self._aggregate_group_tree_metadata(all_fks, group_tree_md)
+
+        if status:
+            status.complete += 1
+            self.status_controller.update(status)
+
     @status_notify(status_type=ImportStatusTypes.AGGREGATE_TAGS)
     def get_aggregate_metadata(
         self,
@@ -155,30 +206,13 @@ class AggregateMetadataMixin(CleanMetadataMixin):
         all_m2m_mds = metadata["m2m_mds"]
         all_fks = metadata["fks"]
         all_failed_imports = metadata["fis"]
-        all_fks.update(
-            {
-                "group_trees": {Publisher: {}, Imprint: {}, Series: {}, Volume: {}},
-            }
-        )
-        for path in all_paths:
-            path_str = str(path)
-            md, m2m_md, group_tree_md, failed_import = self._get_path_metadata(path_str)
+        all_fks.update(self._GROUP_TREES_INIT)
 
-            if failed_import:
-                all_failed_imports.update(failed_import)
-            else:
-                if md:
-                    all_mds[path_str] = md
-
-                if m2m_md:
-                    self._aggregate_m2m_metadata(all_m2m_mds, m2m_md, all_fks, path_str)
-
-                if group_tree_md:
-                    self._aggregate_group_tree_metadata(all_fks, group_tree_md)
-
-            if status:
-                status.complete += 1
-                self.status_controller.update(status)
+        if status and status.complete is None:
+            status.complete = 0
+        data = (all_failed_imports, all_mds, all_m2m_mds, all_fks, status)
+        for path in all_paths:
+            self._aggregate_path(data, path)
 
         all_fks["comic_paths"] = frozenset(all_mds.keys())
         fi_status = Status(ImportStatusTypes.FAILED_IMPORTS, 0, len(all_failed_imports))

codex/librarian/importer/clean_metadata.py

@@ -21,10 +21,10 @@ _MD_INVALID_KEYS = frozenset(
         "updated_at",
     )
 )
-_MD_TRANFORMED_KEYS = frozenset(("credits",))
+_MD_TRANSFORMED_KEYS = frozenset(("credits", "story_arcs"))
 _MD_VALID_KEYS = (
     frozenset([field.name for field in Comic._meta.get_fields()]) - _MD_INVALID_KEYS
-    | _MD_TRANFORMED_KEYS
+    | _MD_TRANSFORMED_KEYS
 )
 _MD_DECIMAL_KEYS = frozenset(("community_rating", "critical_rating"))
 _MD_PSI_KEYS = frozenset(
@@ -33,6 +33,7 @@ _MD_PSI_KEYS = frozenset(
         "month",
         "day",
         "page_count",
+        "story_arc_number",
     )  # computed by presave: ("decade", "max_page", "size")
 )
 _MD_CHAR_KEYS = frozenset(
@@ -42,6 +43,7 @@ _MD_CHAR_KEYS = frozenset(
         "cover_pk",
         "file_type",
         "format",
+        "gtin",
         "issue_suffix",
         "language",
         "name",
@@ -59,7 +61,6 @@ _M2M_NAMED_KEYS = frozenset(
         "genres",
         "locations",
         "series_groups",
-        "story_arcs",
         "tags",
         "teams",
     )
@@ -134,6 +135,16 @@ class CleanMetadataMixin(QueuedThread):
             summary += description
             md["summary"] = summary
 
+    @staticmethod
+    def _append_review(md) -> None:
+        """Append the CIX review to the CIX summary."""
+        if review := md.get("review", ""):
+            summary: str = md.get("summary", "")
+            if summary:
+                summary += "\n"
+            summary += review
+            md["summary"] = summary
+
     @staticmethod
     def _title_to_name(md):
         """Convert title to name for comics."""
@@ -174,13 +185,23 @@ class CleanMetadataMixin(QueuedThread):
             good_creators = []
             field: CharField = NamedModel._meta.get_field("name")  # type:ignore
             for creator in creators:
-                person = cls._clean_charfield(creator.get("person"), field)
-                if person:
+                if person := cls._clean_charfield(creator.get("person"), field):
                     role = cls._clean_charfield(creator.get("role"), field)
                     good_creator = {"role": role, "person": person}
                     good_creators.append(good_creator)
             md["creators"] = good_creators
 
+    @classmethod
+    def _clean_comic_story_arcs(cls, md):
+        """Replace story_arcs with good story_arc_numbers."""
+        if story_arc_numbers := md.pop("story_arcs", None):
+            good_story_arc_numbers = {}
+            field: CharField = NamedModel._meta.get_field("name")  # type:ignore
+            for story_arc_name, number in story_arc_numbers.items():
+                if good_story_arc_name := cls._clean_charfield(story_arc_name, field):
+                    good_story_arc_numbers[good_story_arc_name] = number
+            md["story_arc_numbers"] = good_story_arc_numbers
+
     @staticmethod
     def _clean_comic_web(md):
         """URL field is a special charfield."""
@@ -194,14 +215,14 @@ class CleanMetadataMixin(QueuedThread):
     @classmethod
     def _clean_comic_m2m_named(cls, md: dict[str, Any], md_keys: frozenset[str]):
         """Clean the named models in the m2m fields."""
+        named_field: CharField = NamedModel._meta.get_field("name")  # type:ignore
         for key in _M2M_NAMED_KEYS.intersection(md_keys):
             names = md.get(key)
             if not names:
                 continue
             cleaned_names = []
-            field: CharField = NamedModel._meta.get_field("name")  # type:ignore
             for name in names:
-                cleaned_name = cls._clean_charfield(name, field)
+                cleaned_name = cls._clean_charfield(name, named_field)
                 if cleaned_name:
                     cleaned_names.append(cleaned_name)
             md[key] = cleaned_names
@@ -228,6 +249,8 @@ class CleanMetadataMixin(QueuedThread):
         self._clean_comic_charfields(md, md_keys)
         self._clean_comic_web(md)
         self._append_description(md)
+        self._append_review(md)
         self._clean_comic_creators(md)
+        self._clean_comic_story_arcs(md)
         self._clean_comic_m2m_named(md, md_keys)
         return md