codex 1.7.14__py3-none-any.whl → 1.8.0__py3-none-any.whl

codex/librarian/scribe/importer/query/update_fks.py

@@ -0,0 +1,140 @@
+"""Query the missing foreign keys methods."""
+
+from comicbox.enums.comicbox import compare_identifier_source
+
+from codex.librarian.scribe.importer.const import MODEL_REL_MAP
+from codex.librarian.scribe.importer.query.filters import (
+    QueryForeignKeysFilterImporter,
+)
+from codex.models.base import BaseModel
+
+
+class QueryIsUpdateImporter(QueryForeignKeysFilterImporter):
+    """Query the missing foreign keys methods."""
+
+    @staticmethod
+    def _query_missing_models_is_do_update_identifier(
+        best_extra_values: tuple,
+        proposed_extra_values: tuple,
+        synthesized_extra_proposed_values: list,
+    ):
+        """Compare Identifiers."""
+        best_identifier = best_extra_values[0]
+        if proposed_identifier := proposed_extra_values[0]:
+            if best_identifier:
+                best_id_source, _, best_id_key = best_identifier
+                (
+                    proposed_id_source,
+                    _,
+                    proposed_id_key,
+                ) = proposed_identifier
+
+                if (
+                    compare_identifier_source(best_id_source, proposed_id_source)
+                    or best_id_key != proposed_id_key
+                ):
+                    best_identifier = proposed_identifier
+            else:
+                best_identifier = proposed_identifier
+
+        synthesized_extra_proposed_values.append(best_identifier)
+
+    @staticmethod
+    def _query_missing_models_is_do_update_extra(
+        best_extra_values: tuple,
+        proposed_extra_values: tuple,
+        synthesized_extra_proposed_values: list,
+        *,
+        has_id: bool,
+    ):
+        """Compare extra relations."""
+        if has_id:
+            best_extra_values = best_extra_values[1:]
+            proposed_extra_values = proposed_extra_values[1:]
+        synthesized_extra_values = []
+
+        for best_extra_value, proposed_extra_value in zip(
+            best_extra_values, proposed_extra_values, strict=True
+        ):
+            if (
+                proposed_extra_value is not None
+                and proposed_extra_value != best_extra_value
+            ):
+                synthesized_extra_values.append(proposed_extra_value)
+            else:
+                synthesized_extra_values.append(best_extra_value)
+
+        synthesized_extra_proposed_values.extend(synthesized_extra_values)
+
+    @staticmethod
+    def _query_normalize_existing_values(
+        existing_values_tuple: tuple, extra_index: int
+    ):
+        existing_identifier = existing_values_tuple[extra_index : extra_index + 3]
+        if not any(existing_identifier):
+            existing_identifier = None
+        existing_extra_values = existing_values_tuple[extra_index + 3 :]
+        return (existing_identifier, *existing_extra_values)
+
+    @classmethod
+    def _query_update_init_best_and_existing_values(
+        cls,
+        existing_extra_values_tuple: tuple | None,
+        proposed_extra_values_tuples_set: set[tuple],
+    ) -> tuple[tuple | None, tuple]:
+        """
+        Initialize first best value.
+
+        Normalize existing values and assign to best value OR initialize best values
+        with first proposed value.
+        """
+        if existing_extra_values_tuple:
+            best_extra_values = existing_extra_values_tuple
+        else:
+            try:
+                best_extra_values = proposed_extra_values_tuples_set.pop()
+            except KeyError:
+                best_extra_values = ()
+        return existing_extra_values_tuple, best_extra_values
+
+    @classmethod
+    def query_model_best_extra_values(
+        cls,
+        model: type[BaseModel],
+        existing_extra_values_tuple: tuple | None,
+        proposed_extra_values_tuples_set: set[tuple],
+    ) -> tuple[bool, tuple]:
+        """Find possible updates from existing."""
+        if not existing_extra_values_tuple and not proposed_extra_values_tuples_set:
+            return False, ()
+        _, id_rel, *extra_rels = MODEL_REL_MAP[model]
+        existing_extra_values, best_extra_values = (
+            cls._query_update_init_best_and_existing_values(
+                existing_extra_values_tuple,
+                proposed_extra_values_tuples_set,
+            )
+        )
+
+        has_id = bool(id_rel)
+        for extra_values_tuple in proposed_extra_values_tuples_set:
+            synthesized_extra_proposed_values = []
+            if has_id:
+                cls._query_missing_models_is_do_update_identifier(
+                    best_extra_values,
+                    extra_values_tuple,
+                    synthesized_extra_proposed_values,
+                )
+
+            if extra_rels:
+                cls._query_missing_models_is_do_update_extra(
+                    best_extra_values,
+                    extra_values_tuple,
+                    synthesized_extra_proposed_values,
+                    has_id=has_id,
+                )
+
+            best_extra_values = tuple(synthesized_extra_proposed_values)
+
+        do_update = best_extra_values != existing_extra_values
+
+        return do_update, best_extra_values

codex/librarian/scribe/importer/read/__init__.py

@@ -0,0 +1,14 @@
+"""Extract and Aggregate metadata from comic archive."""
+
+from codex.librarian.scribe.importer.read.extract import (
+    ExtractMetadataImporter,
+)
+
+
+class ReadMetadataImporter(ExtractMetadataImporter):
+    """Extract and Aggregate metadata from comics to prepare for importing."""
+
+    def read(self):
+        """Extract and aggregate metadata."""
+        self.extract_metadata()
+        self.aggregate_metadata()

codex/librarian/scribe/importer/read/aggregate_path.py

@@ -0,0 +1,110 @@
+"""Aggregate metadata from comics to prepare for importing."""
+
+from comicbox.schemas.comicbox import (
+    COVER_DATE_KEY,
+    DATE_KEY,
+    NUMBER_KEY,
+    STORE_DATE_KEY,
+    SUFFIX_KEY,
+    TITLE_KEY,
+)
+
+from codex.librarian.scribe.importer.const import (
+    CREATE_COMICS,
+    EXTRACTED,
+    FIS,
+    LINK_FKS,
+    LINK_M2MS,
+    QUERY_MODELS,
+)
+from codex.librarian.scribe.importer.read.folders import AggregatePathMetadataImporter
+from codex.librarian.scribe.importer.statii.failed import (
+    ImporterFailedImportsQueryStatus,
+)
+from codex.librarian.scribe.importer.statii.read import ImporterAggregateStatus
+
+_UNUSED_COMICBOX_FIELDS = (
+    "alternate_images",
+    "bookmark",
+    "credit_primaries",
+    "ext",
+    "manga",
+    "pages",
+    "prices",
+    "remainders",
+    "reprints",
+    "updated_at",
+)
+
+
+class AggregateMetadataImporter(AggregatePathMetadataImporter):
+    """Aggregate metadata from comics to prepare for importing."""
+
+    @staticmethod
+    def _transform_metadata(md):
+        for key in _UNUSED_COMICBOX_FIELDS:
+            md.pop(key, None)
+
+        if date := md.pop(DATE_KEY, None):
+            date.pop(COVER_DATE_KEY, None)
+            date.pop(STORE_DATE_KEY, None)
+            md.update(date)
+
+        if issue := md.pop("issue", None):
+            if number := issue.pop(NUMBER_KEY, None):
+                md["issue_number"] = number
+            if suffix := issue.pop(SUFFIX_KEY, None):
+                md["issue_suffix"] = suffix
+
+        if title := md.pop(TITLE_KEY, None):
+            md["name"] = title
+
+    def _aggregate_path(self, path, status):
+        """Aggregate metadata for one path."""
+        # Prepare
+        md = self.metadata[EXTRACTED].pop(path)
+        self._transform_metadata(md)
+
+        # Aggregate
+        self.metadata[LINK_FKS][path] = {}
+        self.get_fk_metadata(md, path)
+        self.get_m2m_metadata(md, path)
+        if md:
+            self.get_path_metadata(md, path)
+        self.metadata[CREATE_COMICS][str(path)] = md
+
+        # Status
+        status.increment_complete()
+        self.status_controller.update(status)
+
+    def aggregate_metadata(
+        self,
+    ):
+        """Get aggregated metadata for the paths given."""
+        num_extracted_paths = len(self.metadata[EXTRACTED])
+        self.log.debug(
+            f"Aggregating tags from {num_extracted_paths} comics in {self.library.path}..."
+        )
+        status = ImporterAggregateStatus(0, num_extracted_paths)
+        self.status_controller.start(status)
+
+        # Init metadata, extract and aggregate
+        self.metadata[QUERY_MODELS] = {}
+        self.metadata[CREATE_COMICS] = {}
+        self.metadata[LINK_FKS] = {}
+        self.metadata[LINK_M2MS] = {}
+        # Aggregate further
+
+        for path in tuple(self.metadata[EXTRACTED]):
+            if self.abort_event.is_set():
+                return status.complete
+            self._aggregate_path(path, status)
+        del self.metadata[EXTRACTED]
+
+        fis = self.metadata[FIS].keys()
+
+        # Set statii
+        fi_status = ImporterFailedImportsQueryStatus(0, len(fis))
+        self.status_controller.update(fi_status, notify=False)
+        self.status_controller.finish(status)
+        return status.complete

codex/librarian/scribe/importer/read/const.py

@@ -0,0 +1,144 @@
+"""Aggregate Consts."""
+
+from types import MappingProxyType
+
+from comicbox.schemas.comicbox import (
+    ARCS_KEY,
+    CHARACTERS_KEY,
+    DESIGNATION_KEY,
+    GENRES_KEY,
+    ID_KEY_KEY,
+    ID_URL_KEY,
+    IDENTIFIERS_KEY,
+    LOCATIONS_KEY,
+    NUMBER_KEY,
+    PROTAGONIST_KEY,
+    ROLES_KEY,
+    SERIES_GROUPS_KEY,
+    STORIES_KEY,
+    TAGS_KEY,
+    TEAMS_KEY,
+)
+from django.db.models.fields import CharField, Field
+from django.db.models.fields.related_descriptors import ForwardManyToOneDescriptor
+from django.db.models.query_utils import DeferredAttribute
+
+from codex.librarian.scribe.importer.const import (
+    ALL_COMIC_FK_FIELDS,
+    CREDITS_FIELD_NAME,
+    IDENTIFIERS_FIELD_NAME,
+    NAME_FIELD_NAME,
+    PROTAGONIST_FIELD_MODEL_MAP,
+    STORY_ARC_NUMBERS_FIELD_NAME,
+    UNIVERSES_FIELD_NAME,
+)
+from codex.models import (
+    StoryArc,
+    StoryArcNumber,
+    Universe,
+)
+from codex.models.identifier import (
+    Identifier,
+    IdentifierSource,
+)
+from codex.models.named import (
+    Character,
+    CreditPerson,
+    CreditRole,
+    Genre,
+    Location,
+    SeriesGroup,
+    Story,
+    Tag,
+    Team,
+)
+
+######
+# FK #
+######
+_EXCLUDE_FIELD_NAMES = frozenset(PROTAGONIST_FIELD_MODEL_MAP.keys() | {"parent_folder"})
+COMIC_FK_FIELD_NAMES_FIELD_MAP: MappingProxyType[str, Field] = MappingProxyType(
+    {
+        **{
+            field.name: field.related_model._meta.get_field(NAME_FIELD_NAME)
+            for field in ALL_COMIC_FK_FIELDS
+            if field.related_model and field.name not in _EXCLUDE_FIELD_NAMES
+        },
+        PROTAGONIST_KEY: PROTAGONIST_FIELD_MODEL_MAP["main_character"]._meta.get_field(
+            NAME_FIELD_NAME
+        ),
+    }
+)
+
+#######
+# M2M #
+#######
+FIELD_NAME_TO_MD_KEY_MAP = MappingProxyType(
+    {
+        STORY_ARC_NUMBERS_FIELD_NAME: ARCS_KEY,
+    }
+)
+SIMPLE_KEY_CLASS_MAP = MappingProxyType(
+    {
+        SERIES_GROUPS_KEY: SeriesGroup,
+    }
+)
+IDENTIFIED_KEY_CLASS_MAP = MappingProxyType(
+    {
+        CHARACTERS_KEY: Character,
+        GENRES_KEY: Genre,
+        LOCATIONS_KEY: Location,
+        STORIES_KEY: Story,
+        TAGS_KEY: Tag,
+        TEAMS_KEY: Team,
+    }
+)
+ID_TYPE_KEY = "id_type"
+# This map tells aggregator how to parse metadata into tuples for query & create.
+COMPLEX_FIELD_AGG_MAP: MappingProxyType[
+    str,
+    tuple[
+        DeferredAttribute | CharField,
+        ForwardManyToOneDescriptor | None,
+        dict[str, DeferredAttribute],
+    ],
+] = MappingProxyType(
+    {
+        **{key: (cls.name, None, {}) for key, cls in SIMPLE_KEY_CLASS_MAP.items()},
+        **{
+            key: (cls.name, cls.identifier, {})
+            for key, cls in IDENTIFIED_KEY_CLASS_MAP.items()
+        },
+        CREDITS_FIELD_NAME: (
+            CreditPerson.name,
+            CreditPerson.identifier,
+            {
+                ROLES_KEY: CreditRole.name,
+            },
+        ),
+        IDENTIFIERS_FIELD_NAME: (
+            IdentifierSource.name,
+            None,
+            {
+                ID_TYPE_KEY: "comic",
+                ID_KEY_KEY: Identifier.key,
+                ID_URL_KEY: Identifier.url,
+            },
+        ),
+        STORY_ARC_NUMBERS_FIELD_NAME: (
+            StoryArc.name,
+            StoryArc.identifier,
+            {
+                NUMBER_KEY: StoryArcNumber.number,
+            },
+        ),
+        UNIVERSES_FIELD_NAME: (
+            Universe.name,
+            None,
+            {
+                IDENTIFIERS_KEY: Universe.identifier,
+                DESIGNATION_KEY: Universe.designation,
+            },
+        ),
+    }
+)

codex/librarian/scribe/importer/read/extract.py

@@ -0,0 +1,128 @@
+"""Extract metadata from comic archive."""
+
+from datetime import datetime
+from tarfile import TarError
+from zipfile import BadZipFile, LargeZipFile
+
+from comicbox.box import Comicbox
+from comicbox.exceptions import UnsupportedArchiveTypeError
+from py7zr.exceptions import ArchiveError as Py7zError
+from rarfile import Error as RarError
+
+from codex.choices.admin import AdminFlagChoices
+from codex.librarian.scribe.importer.const import EXTRACTED, FIS, SKIPPED
+from codex.librarian.scribe.importer.read.aggregate_path import (
+    AggregateMetadataImporter,
+)
+from codex.librarian.scribe.importer.statii.read import ImporterReadComicsStatus
+from codex.models.admin import AdminFlag
+from codex.models.comic import Comic
+from codex.settings import COMICBOX_CONFIG
+
+
+class ExtractMetadataImporter(AggregateMetadataImporter):
+    """Aggregate metadata from comics to prepare for importing."""
+
+    @staticmethod
+    def _metadata_mtime(path: str) -> datetime | None:
+        try:
+            comic = Comic.objects.get(path=path)
+        except Comic.DoesNotExist:
+            return None
+        return comic.metadata_mtime
+
+    def _set_import_metadata_flag(self):
+        """Set import_metadata flag."""
+        if self.task.force_import_metadata:
+            import_metadata = True
+        else:
+            key = AdminFlagChoices.IMPORT_METADATA.value
+            import_metadata = AdminFlag.objects.only("on").get(key=key).on
+        if not import_metadata:
+            self.log.warning("Admin flag set to NOT import metadata.")
+        return import_metadata
+
+    def _extract_path(self, path: str, *, import_metadata: bool):
+        """Extract metadata from comic and clean it for codex."""
+        md = {}
+        failed_import = {}
+        try:
+            if import_metadata:
+                with Comicbox(path, config=COMICBOX_CONFIG, logger=self.log) as cb:
+                    new_md_mtime = cb.get_metadata_mtime()
+                    if self.task.check_metadata_mtime:
+                        old_md_mtime = self._metadata_mtime(path)
+                        if (
+                            old_md_mtime
+                            and new_md_mtime
+                            and new_md_mtime <= old_md_mtime
+                        ):
+                            return md
+                    md = cb.to_dict()
+                    md = md.get("comicbox", {})
+                    md["file_type"] = cb.get_file_type()
+                    md["metadata_mtime"] = new_md_mtime
+            md["path"] = path
+        except (
+            UnsupportedArchiveTypeError,
+            BadZipFile,
+            LargeZipFile,
+            RarError,
+            Py7zError,
+            TarError,
+            OSError,
+        ) as exc:
+            self.log.warning(f"Failed to import {path}: {exc}")
+            failed_import = {path: exc}
+        except Exception as exc:
+            self.log.exception(f"Failed to import: {path}")
+            failed_import = {path: exc}
+        if failed_import:
+            self.metadata[FIS].update(failed_import)
+        return md
+
+    def extract_metadata(self, status=None) -> int:
+        """Extract comic metadata into memory."""
+        count = 0
+        self.metadata[SKIPPED] = set()
+        self.metadata[EXTRACTED] = {}
+        self.metadata[FIS] = {}
+        all_paths = self.task.files_modified | self.task.files_created
+        self.task.files_modified = frozenset()
+        self.task.files_created = frozenset()
+        total_paths = len(all_paths)
+        status = ImporterReadComicsStatus(0, total_paths)
+        try:
+            if not total_paths:
+                return count
+
+            self.log.debug(
+                f"Reading tags from {total_paths} comics in {self.library.path}..."
+            )
+            self.status_controller.start(status, notify=True)
+
+            # Set import_metadata flag
+            import_metadata = self._set_import_metadata_flag()
+
+            for path in all_paths:
+                if self.abort_event.is_set():
+                    return count
+                if md := self._extract_path(path, import_metadata=import_metadata):
+                    self.metadata[EXTRACTED][path] = md
+                else:
+                    self.metadata[SKIPPED].add(path)
+
+                status.increment_complete()
+                self.status_controller.update(status)
+
+            skipped_count = len(self.metadata[SKIPPED])
+            count = total_paths - skipped_count
+            level = "INFO" if skipped_count else "DEBUG"
+            self.log.log(
+                level,
+                f"Skipped {skipped_count} comics because metadata appears unchanged.",
+            )
+        finally:
+            self.metadata.pop(SKIPPED)
+            self.status_controller.finish(status)
+        return count

codex/librarian/scribe/importer/read/folders.py

@@ -0,0 +1,37 @@
+"""Aggregate folders and path from path."""
+
+from pathlib import Path
+
+from codex.librarian.scribe.importer.const import (
+    FOLDERS_FIELD_NAME,
+    LINK_M2MS,
+    PATH_FIELD_NAME,
+)
+from codex.librarian.scribe.importer.read.many_to_many import (
+    AggregateManyToManyMetadataImporter,
+)
+from codex.models.groups import Folder
+
+
+class AggregatePathMetadataImporter(AggregateManyToManyMetadataImporter):
+    """Aggregate path metadata."""
+
+    def get_all_library_relative_paths(self, comic_paths):
+        """Get the proposed folder_paths."""
+        # also used by moved/comic.py:_bulk_comics_moved_ensure_folders()
+        library_path = Path(self.library.path)
+        proposed_folder_paths = set()
+
+        for comic_path in comic_paths:
+            for path in Path(comic_path).parents:
+                if path.is_relative_to(library_path):
+                    proposed_folder_paths.add((str(path),))
+        return proposed_folder_paths
+
+    def get_path_metadata(self, md: dict, path: Path | str):
+        """Set the path metadata."""
+        proposed_folder_paths = self.get_all_library_relative_paths((path,))
+        for proposed_path in proposed_folder_paths:
+            self.add_query_model(Folder, proposed_path)
+        self.metadata[LINK_M2MS][path][FOLDERS_FIELD_NAME] = proposed_folder_paths
+        md[PATH_FIELD_NAME] = str(path)