codex 1.7.15__py3-none-any.whl → 1.8.0a2__py3-none-any.whl

codex/librarian/scribe/importer/read/extract.py

@@ -0,0 +1,128 @@
+"""Extract metadata from comic archive."""
+
+from datetime import datetime
+from tarfile import TarError
+from zipfile import BadZipFile, LargeZipFile
+
+from comicbox.box import Comicbox
+from comicbox.exceptions import UnsupportedArchiveTypeError
+from py7zr.exceptions import ArchiveError as Py7zError
+from rarfile import Error as RarError
+
+from codex.choices.admin import AdminFlagChoices
+from codex.librarian.scribe.importer.const import EXTRACTED, FIS, SKIPPED
+from codex.librarian.scribe.importer.read.aggregate_path import (
+    AggregateMetadataImporter,
+)
+from codex.librarian.scribe.importer.statii.read import ImporterReadComicsStatus
+from codex.models.admin import AdminFlag
+from codex.models.comic import Comic
+from codex.settings import COMICBOX_CONFIG
+
+
+class ExtractMetadataImporter(AggregateMetadataImporter):
+    """Aggregate metadata from comics to prepare for importing."""
+
+    @staticmethod
+    def _metadata_mtime(path: str) -> datetime | None:
+        try:
+            comic = Comic.objects.get(path=path)
+        except Comic.DoesNotExist:
+            return None
+        return comic.metadata_mtime
+
+    def _set_import_metadata_flag(self):
+        """Set import_metadata flag."""
+        if self.task.force_import_metadata:
+            import_metadata = True
+        else:
+            key = AdminFlagChoices.IMPORT_METADATA.value
+            import_metadata = AdminFlag.objects.only("on").get(key=key).on
+        if not import_metadata:
+            self.log.warning("Admin flag set to NOT import metadata.")
+        return import_metadata
+
+    def _extract_path(self, path: str, *, import_metadata: bool):
+        """Extract metadata from comic and clean it for codex."""
+        md = {}
+        failed_import = {}
+        try:
+            if import_metadata:
+                with Comicbox(path, config=COMICBOX_CONFIG, logger=self.log) as cb:
+                    new_md_mtime = cb.get_metadata_mtime()
+                    if self.task.check_metadata_mtime:
+                        old_md_mtime = self._metadata_mtime(path)
+                        if (
+                            old_md_mtime
+                            and new_md_mtime
+                            and new_md_mtime <= old_md_mtime
+                        ):
+                            return md
+                    md = cb.to_dict()
+                    md = md.get("comicbox", {})
+                    md["file_type"] = cb.get_file_type()
+                    md["metadata_mtime"] = new_md_mtime
+            md["path"] = path
+        except (
+            UnsupportedArchiveTypeError,
+            BadZipFile,
+            LargeZipFile,
+            RarError,
+            Py7zError,
+            TarError,
+            OSError,
+        ) as exc:
+            self.log.warning(f"Failed to import {path}: {exc}")
+            failed_import = {path: exc}
+        except Exception as exc:
+            self.log.exception(f"Failed to import: {path}")
+            failed_import = {path: exc}
+        if failed_import:
+            self.metadata[FIS].update(failed_import)
+        return md
+
+    def extract_metadata(self, status=None) -> int:
+        """Extract comic metadata into memory."""
+        count = 0
+        self.metadata[SKIPPED] = set()
+        self.metadata[EXTRACTED] = {}
+        self.metadata[FIS] = {}
+        all_paths = self.task.files_modified | self.task.files_created
+        self.task.files_modified = frozenset()
+        self.task.files_created = frozenset()
+        total_paths = len(all_paths)
+        status = ImporterReadComicsStatus(0, total_paths)
+        try:
+            if not total_paths:
+                return count
+
+            self.log.debug(
+                f"Reading tags from {total_paths} comics in {self.library.path}..."
+            )
+            self.status_controller.start(status, notify=True)
+
+            # Set import_metadata flag
+            import_metadata = self._set_import_metadata_flag()
+
+            for path in all_paths:
+                if self.abort_event.is_set():
+                    return count
+                if md := self._extract_path(path, import_metadata=import_metadata):
+                    self.metadata[EXTRACTED][path] = md
+                else:
+                    self.metadata[SKIPPED].add(path)
+
+                status.increment_complete()
+                self.status_controller.update(status)
+
+            skipped_count = len(self.metadata[SKIPPED])
+            count = total_paths - skipped_count
+            level = "INFO" if skipped_count else "DEBUG"
+            self.log.log(
+                level,
+                f"Skipped {skipped_count} comics because metadata appears unchanged.",
+            )
+        finally:
+            self.metadata.pop(SKIPPED)
+            self.status_controller.finish(status)
+        return count

codex/librarian/scribe/importer/read/folders.py

@@ -0,0 +1,37 @@
+"""Aggregate folders and path from path."""
+
+from pathlib import Path
+
+from codex.librarian.scribe.importer.const import (
+    FOLDERS_FIELD_NAME,
+    LINK_M2MS,
+    PATH_FIELD_NAME,
+)
+from codex.librarian.scribe.importer.read.many_to_many import (
+    AggregateManyToManyMetadataImporter,
+)
+from codex.models.groups import Folder
+
+
+class AggregatePathMetadataImporter(AggregateManyToManyMetadataImporter):
+    """Aggregate path metadata."""
+
+    def get_all_library_relative_paths(self, comic_paths):
+        """Get the proposed folder_paths."""
+        # also used by moved/comic.py:_bulk_comics_moved_ensure_folders()
+        library_path = Path(self.library.path)
+        proposed_folder_paths = set()
+
+        for comic_path in comic_paths:
+            for path in Path(comic_path).parents:
+                if path.is_relative_to(library_path):
+                    proposed_folder_paths.add((str(path),))
+        return proposed_folder_paths
+
+    def get_path_metadata(self, md: dict, path: Path | str):
+        """Set the path metadata."""
+        proposed_folder_paths = self.get_all_library_relative_paths((path,))
+        for proposed_path in proposed_folder_paths:
+            self.add_query_model(Folder, proposed_path)
+        self.metadata[LINK_M2MS][path][FOLDERS_FIELD_NAME] = proposed_folder_paths
+        md[PATH_FIELD_NAME] = str(path)

codex/librarian/scribe/importer/read/foreign_keys.py

@@ -0,0 +1,145 @@
+"""Aggregate Browser Group Trees."""
+
+from collections.abc import Mapping
+from contextlib import suppress
+
+from comicbox.identifiers import IdSources
+from comicbox.schemas.comicbox import (
+    ID_KEY_KEY,
+    ID_URL_KEY,
+    IDENTIFIERS_KEY,
+    NAME_KEY,
+    NUMBER_KEY,
+    NUMBER_TO_KEY,
+    PROTAGONIST_KEY,
+)
+from django.db.models import Field
+from django.db.models.base import Model
+
+from codex.librarian.scribe.importer.const import (
+    GROUP_MODEL_COUNT_FIELDS,
+    LINK_FKS,
+    QUERY_MODELS,
+)
+from codex.librarian.scribe.importer.query import QueryForeignKeysImporter
+from codex.librarian.scribe.importer.read.const import COMIC_FK_FIELD_NAMES_FIELD_MAP
+from codex.models.base import BaseModel
+from codex.models.groups import BrowserGroupModel, Volume
+from codex.models.identifier import Identifier, IdentifierSource
+from codex.util import max_none
+
+
+class AggregateForeignKeyMetadataImporter(QueryForeignKeysImporter):
+    """Aggregate Browser Group Trees."""
+
+    def add_query_model(
+        self,
+        model: type[Model],
+        clean_key_values: tuple,
+        clean_extra_values: frozenset | set | None = None,
+    ):
+        """Add to the queury models set for the model."""
+        if model not in self.metadata[QUERY_MODELS]:
+            self.metadata[QUERY_MODELS][model] = {}
+        if clean_key_values not in self.metadata[QUERY_MODELS][model]:
+            self.metadata[QUERY_MODELS][model][clean_key_values] = set()
+        if clean_extra_values is None:
+            clean_extra_values = frozenset()
+        else:
+            clean_extra_values = frozenset(clean_extra_values)
+        self.metadata[QUERY_MODELS][model][clean_key_values] |= clean_extra_values
+
+    def get_identifier_tuple(self, model: type[BaseModel], obj: Mapping):
+        """Parse first highest priority identifier from metadata."""
+        # Used by Objects with identifiers, not comic itself.
+        identifiers = obj.get(IDENTIFIERS_KEY)
+        if not identifiers:
+            return None
+
+        for id_source_enum in IdSources:
+            id_source = id_source_enum.value
+            if id_obj := identifiers.get(id_source):
+                break
+        else:
+            id_source, id_obj = next(identifiers.items())
+
+        if not id_obj:
+            return None
+
+        id_type = model._meta.db_table.removeprefix("codex_")
+
+        id_key = id_obj.get(ID_KEY_KEY)
+        if not id_key:
+            return None
+        id_url = id_obj.get(ID_URL_KEY)
+
+        identifier_tuple_keys = (id_source, id_type, id_key)
+        identifier_tuple_extra = frozenset([(id_url,)])
+        self.add_query_model(IdentifierSource, (id_source,))
+        self.add_query_model(Identifier, identifier_tuple_keys, identifier_tuple_extra)
+
+        return identifier_tuple_keys
+
+    def _set_simple_fk(self, related_field: Field, value) -> tuple:
+        value = related_field.get_prep_value(value)
+        if value is None:
+            return ()
+        return (value,), None
+
+    def _set_group_tree_group(
+        self,
+        model: type[BrowserGroupModel],
+        name_field: Field,
+        group: dict | None,
+        group_list: list,
+    ) -> tuple:
+        name_key = NUMBER_KEY if model == Volume else NAME_KEY
+        if group is None:
+            group = {}
+
+        group_name = group.get(name_key, model.DEFAULT_NAME)
+        clean_group_name = name_field.get_prep_value(group_name)
+        group_list.append(clean_group_name)
+        extra_vals = []
+        if model == Volume:
+            number_to = group.get(NUMBER_TO_KEY, model.DEFAULT_NAME)
+            clean_number_to = name_field.get_prep_value(number_to)
+            group_list.append(clean_number_to)
+        else:
+            identifier_tuple = self.get_identifier_tuple(model, group)
+            extra_vals.append(identifier_tuple)
+        count_key = GROUP_MODEL_COUNT_FIELDS[model]
+        if count_key:
+            count = group.get(count_key)
+            with suppress(Exception):
+                old_count_val = (
+                    self.metadata[QUERY_MODELS].get(model, {}).get(group_list)
+                )
+                count = max_none(old_count_val, count)
+            extra_vals.append(count)
+        return tuple(group_list), frozenset((tuple(extra_vals),))
+
+    def get_fk_metadata(self, md, path):
+        """Aggregate Simple Foreign Keys."""
+        group_list = []
+        for field_name, related_field in COMIC_FK_FIELD_NAMES_FIELD_MAP.items():
+            # No identifiers on many2one fks yet
+            # md_key = FIELD_NAME_TO_MD_KEY_MAP.get(field_name, field_name) if they ever diverge
+            model: type[BaseModel] = related_field.model  # pyright: ignore[reportAssignmentType]
+            md_key = field_name
+            value = md.pop(md_key, None)
+            if value is None and not issubclass(model, BrowserGroupModel):
+                continue
+
+            if issubclass(model, BrowserGroupModel):
+                key_values, extra_values = self._set_group_tree_group(
+                    model, related_field, value, group_list
+                )
+            else:
+                key_values, extra_values = self._set_simple_fk(related_field, value)
+            if not key_values and not extra_values:
+                continue
+
+            if md_key != PROTAGONIST_KEY:
+                self.add_query_model(model, key_values, extra_values)
+            self.metadata[LINK_FKS][path][field_name] = key_values

codex/librarian/scribe/importer/read/many_to_many.py

@@ -0,0 +1,237 @@
+"""Aggregate ManyToMany Metadata."""
+
+from collections.abc import Mapping
+from typing import TYPE_CHECKING
+
+from comicbox.schemas.comicbox import IDENTIFIERS_KEY, NUMBER_KEY, ROLES_KEY
+from django.db.models import CharField, Field
+from django.db.models.fields.related import ManyToManyField
+
+from codex.librarian.scribe.importer.const import (
+    COMIC_M2M_FIELDS,
+    CREDITS_FIELD_NAME,
+    IDENTIFIERS_FIELD_NAME,
+    LINK_M2MS,
+    STORY_ARC_NUMBERS_FIELD_NAME,
+    get_key_index,
+)
+from codex.librarian.scribe.importer.read.const import (
+    COMPLEX_FIELD_AGG_MAP,
+    FIELD_NAME_TO_MD_KEY_MAP,
+    ID_TYPE_KEY,
+)
+from codex.librarian.scribe.importer.read.foreign_keys import (
+    AggregateForeignKeyMetadataImporter,
+)
+from codex.models.comic import Comic
+from codex.models.groups import Folder
+from codex.models.identifier import IdentifierSource
+from codex.models.named import CreditRole
+
+if TYPE_CHECKING:
+    from codex.models.base import BaseModel
+
+
+class AggregateManyToManyMetadataImporter(AggregateForeignKeyMetadataImporter):
+    """Aggregate ManyToMany Metadata."""
+
+    def _get_m2m_metadata_dict_model_aggregate_sub_sub_value(
+        self,
+        field_name: str,
+        sub_value_obj: Mapping | None,
+        sub_sub_md_key: str,
+        sub_sub_field: Field,
+    ):
+        # sub_sub_md_key is identifiers or designation or roles or number
+        # StoryArcNumber.number can be None.
+        clean_sub_sub_values = None
+        if sub_value_obj is None:
+            return clean_sub_sub_values
+        sub_sub_value = sub_value_obj.get(sub_sub_md_key)
+        if sub_sub_value is None and field_name != STORY_ARC_NUMBERS_FIELD_NAME:
+            return clean_sub_sub_values
+
+        if sub_sub_md_key == IDENTIFIERS_KEY and sub_sub_value:
+            field = Comic._meta.get_field(field_name)
+            model: type[BaseModel] = field.related_model  # pyright: ignore[reportAssignmentType]
+            clean_sub_sub_values = self.get_identifier_tuple(model, sub_sub_value)
+        elif sub_sub_md_key == ROLES_KEY and sub_sub_value:
+            clean_sub_sub_values = set()
+            for sub_sub_sub_key_name, sub_sub_sub_value_obj in sub_sub_value.items():
+                clean_sub_sub_sub_key_name = sub_sub_field.get_prep_value(
+                    sub_sub_sub_key_name
+                )
+                sub_sub_key_identifier_tuple = self.get_identifier_tuple(
+                    CreditRole, sub_sub_sub_value_obj
+                )
+                clean_sub_sub_values.add(
+                    (clean_sub_sub_sub_key_name, sub_sub_key_identifier_tuple)
+                )
+            clean_sub_sub_values = frozenset(clean_sub_sub_values)
+        else:
+            clean_sub_sub_values = sub_sub_field.get_prep_value(sub_sub_value)
+            if sub_sub_md_key == NUMBER_KEY:
+                clean_sub_sub_values = frozenset({clean_sub_sub_values})
+        return clean_sub_sub_values
+
+    def _get_m2m_metadata_aggregate_sub_values_init(
+        self,
+        md_key: str,
+        sub_key_name_field,
+        sub_key_name: str,
+        sub_key_identifier_field,
+        sub_value_obj,
+    ):
+        name_field = (
+            sub_key_name_field
+            if isinstance(sub_key_name_field, CharField)
+            else sub_key_name_field.field
+        )
+        clean_sub_key_name = name_field.get_prep_value(sub_key_name)
+
+        clean_sub_values = []
+        sub_model = None
+        if sub_key_identifier_field:
+            sub_model = sub_key_identifier_field.field.model
+            sub_key_identifier_tuple = self.get_identifier_tuple(
+                sub_model, sub_value_obj
+            )
+            clean_sub_values.append(sub_key_identifier_tuple)
+        elif md_key == IDENTIFIERS_KEY and clean_sub_key_name:
+            sub_model = IdentifierSource
+
+        clean_sub_key = (clean_sub_key_name,)
+        if sub_model:
+            clean_sub_values_set = (
+                frozenset({tuple(clean_sub_values)}) if clean_sub_values else None
+            )
+            self.add_query_model(sub_model, clean_sub_key, clean_sub_values_set)
+        return clean_sub_key, clean_sub_values
+
+    def _get_m2m_metadata_dict_model_aggregate_sub_values(
+        self,
+        md_key: str,
+        field: ManyToManyField,
+        sub_key_name: str,
+        sub_value_obj: Mapping | None,
+    ):
+        # Clean name and if there are sub values get those.
+        # sub_key: story_arc_name_a,
+        # sub_key: character_name_a,
+        sub_key_name_field, sub_key_identifier_field, dict_field_keys = (
+            COMPLEX_FIELD_AGG_MAP[field.name]
+        )
+        clean_sub_key, clean_sub_values = (
+            self._get_m2m_metadata_aggregate_sub_values_init(
+                md_key,
+                sub_key_name_field,
+                sub_key_name,
+                sub_key_identifier_field,
+                sub_value_obj,
+            )
+        )
+
+        roles_or_numbers = set()
+        for sub_sub_md_key, sub_sub_field in dict_field_keys.items():
+            # Sub_sub_md_key is identifiers or designation or roles
+
+            if sub_sub_md_key == ID_TYPE_KEY:
+                # Special injection of identifier type
+                clean_sub_values.append(sub_sub_field)
+                continue
+
+            # Get one sub value tuple for the aggregate tuple
+            clean_sub_sub_value = (
+                self._get_m2m_metadata_dict_model_aggregate_sub_sub_value(
+                    field.name,
+                    sub_value_obj,
+                    sub_sub_md_key,
+                    sub_sub_field.field,
+                )
+            )
+            if isinstance(clean_sub_sub_value, frozenset):
+                # Special multiplier for Roles
+                roles_or_numbers |= clean_sub_sub_value
+            else:
+                clean_sub_values.append(clean_sub_sub_value)
+
+        # Create sub_map with special provisions for complex types.
+        clean_sub_map = {}
+        if roles_or_numbers:
+            clean_sub_key = clean_sub_key[0]
+            clean_sub_map = {}
+            if field.name == CREDITS_FIELD_NAME:
+                # Credits
+                for role_values in roles_or_numbers:
+                    role_keys, role_extras = role_values
+                    self.add_query_model(
+                        CreditRole, (role_keys,), frozenset({(role_extras,)})
+                    )
+                    clean_sub_map[(clean_sub_key, role_keys)] = set()
+            else:
+                # StoryArcNumbers
+                for role_values in roles_or_numbers:
+                    clean_sub_map[(clean_sub_key, role_values)] = set()
+        elif field.name == IDENTIFIERS_FIELD_NAME:
+            clean_sub_key += tuple(clean_sub_values[:2])
+            url_value = tuple(clean_sub_values[2:])
+            clean_sub_value = frozenset({url_value})
+            clean_sub_map = {clean_sub_key: clean_sub_value}
+        else:
+            clean_sub_value = (
+                frozenset((tuple(clean_sub_values),))
+                if clean_sub_values
+                else frozenset()
+            )
+            clean_sub_map = {clean_sub_key: clean_sub_value}
+        # Create query models for complex types who's keys are other types.
+        return clean_sub_map
+
+    def _get_m2m_metadata_dict_model(
+        self,
+        md_key: str,
+        field: ManyToManyField,
+        values: Mapping | list | tuple | set | frozenset,
+    ):
+        # Process values dict for a field
+        # {story_arc_name_a: { number: 1, identifiers: {} }, ...}
+        # {character_name_a: { identifiers: {} }, ...}
+        clean_values_map: dict[tuple, frozenset[tuple]] = {}
+        if not isinstance(values, Mapping):
+            values = dict.fromkeys(values)
+        related_model: type[BaseModel] = field.related_model
+        for sub_key_name, sub_value in values.items():
+            clean_sub_map = self._get_m2m_metadata_dict_model_aggregate_sub_values(
+                md_key, field, sub_key_name, sub_value
+            )
+            clean_values_map.update(clean_sub_map)
+
+        if related_model != Folder:
+            for key, value in clean_values_map.items():
+                self.add_query_model(related_model, key, value)
+        return clean_values_map
+
+    def get_m2m_metadata(self, md, path):
+        """Many_to_many fields get moved into a separate dict."""
+        m2m_md = {}
+        for field in COMIC_M2M_FIELDS:
+            md_key = FIELD_NAME_TO_MD_KEY_MAP.get(field.name, field.name)
+            values = md.pop(md_key, None)
+            if values is None:
+                continue
+            if clean_values := self._get_m2m_metadata_dict_model(md_key, field, values):
+                key_index = get_key_index(field.related_model)
+                clean_key_values = set()
+                for clean_val_tuple in clean_values:
+                    clean_key_values_tuple = clean_val_tuple[:key_index]
+                    deep_trimmed_key_values = []
+                    for val in clean_key_values_tuple:
+                        # might have to lok up index in the future but for now it's the default.
+                        deep_trimmed_val = val[0] if isinstance(val, tuple) else val
+                        deep_trimmed_key_values.append(deep_trimmed_val)
+                    clean_key_values.add(tuple(deep_trimmed_key_values))
+                if field.name not in m2m_md:
+                    m2m_md[field.name] = set()
+                m2m_md[field.name] |= clean_key_values
+
+        self.metadata[LINK_M2MS][str(path)] = m2m_md

codex/librarian/scribe/importer/search/__init__.py

@@ -0,0 +1,34 @@
+"""Sync the fts index with the imported database."""
+
+from codex.librarian.scribe.importer.search.prepare import (
+    SearchIndexPrepareImporter,
+)
+from codex.librarian.scribe.importer.statii.search import (
+    ImporterFTSCreateStatus,
+    ImporterFTSUpdateStatus,
+)
+from codex.librarian.scribe.search.handler import SearchIndexer
+from codex.librarian.scribe.search.status import SearchIndexCleanStatus
+
+_STATII = (SearchIndexCleanStatus, ImporterFTSCreateStatus, ImporterFTSUpdateStatus)
+
+
+class SearchIndexImporter(SearchIndexPrepareImporter):
+    """Sync the fts index with the imported database."""
+
+    def clean_fts(self) -> int:
+        """Clean search index of any deleted comics."""
+        indexer = SearchIndexer(
+            self.log, self.librarian_queue, self.db_write_lock, event=self.abort_event
+        )
+        return indexer.remove_stale_records()
+
+    def full_text_search(self):
+        """Sync the fts index with the imported database."""
+        statii = (status_class() for status_class in _STATII)
+        self.status_controller.start_many(statii)
+        try:
+            count = self.clean_fts()
+            self.import_search_index(count)
+        finally:
+            self.status_controller.finish_many(statii)

codex/librarian/scribe/importer/search/const.py

@@ -0,0 +1,50 @@
+"""Search import consts."""
+
+from types import MappingProxyType
+
+from codex.serializers.fields.browser import CountryField, LanguageField
+
+_COMICFTS_ATTRIBUTES = (
+    "collection_title",
+    "file_type",
+    "issue",
+    "name",
+    "notes",
+    "reading_direction",
+    "review",
+    "summary",
+    "updated_at",
+)
+_COMICFTS_FKS = (
+    "publisher",
+    "imprint",
+    "series",
+    "age_rating",
+    "country",
+    "language",
+    "original_format",
+    "scan_info",
+    "tagger",
+)
+_COMICFTS_M2MS = (
+    "characters",
+    "credits",
+    "genres",
+    "identifiers",
+    "locations",
+    "series_groups",
+    "sources",
+    "stories",
+    "story_arcs",
+    "tags",
+    "teams",
+    "universes",
+)
+COMICFTS_UPDATE_FIELDS = (
+    *_COMICFTS_ATTRIBUTES,
+    *_COMICFTS_FKS,
+    *_COMICFTS_M2MS,
+)
+PYCOUNTRY_FIELDS = MappingProxyType(
+    {"country": CountryField(), "language": LanguageField()}
+)