codex 1.7.14__py3-none-any.whl → 1.8.0a2__py3-none-any.whl

codex/librarian/scribe/importer/query/query_fks.py

@@ -0,0 +1,203 @@
+"""Query the missing foreign keys methods."""
+
+from codex.librarian.scribe.importer.const import (
+    CREATE_FKS,
+    FK_KEYS,
+    MODEL_REL_MAP,
+    MODEL_SELECT_RELATED,
+    QUERY_MODELS,
+    TOTAL,
+    UPDATE_FKS,
+    get_key_index,
+)
+from codex.librarian.scribe.importer.query.update_fks import QueryIsUpdateImporter
+from codex.librarian.scribe.importer.statii.create import (
+    ImporterCreateTagsStatus,
+    ImporterUpdateTagsStatus,
+)
+from codex.librarian.scribe.importer.statii.query import ImporterQueryMissingTagsStatus
+from codex.librarian.status import Status
+from codex.models.base import BaseModel
+from codex.models.named import Universe
+from codex.settings import FILTER_BATCH_SIZE
+from codex.util import flatten
+
+
+class QueryForeignKeysQueryImporter(QueryIsUpdateImporter):
+    """Query the missing foreign keys methods."""
+
+    def query_existing_mds(self, model: type[BaseModel], batch_proposed_key_tuples):
+        """Query existing metatata tables."""
+        key_rels: tuple[str, ...] = MODEL_REL_MAP[model][0]  # pyright:ignore[reportAssignmentType]
+        fk_filter = self.query_missing_model_filter(
+            model,
+            key_rels,
+            batch_proposed_key_tuples,
+        )
+        rels = MODEL_REL_MAP[model]
+        select_related = MODEL_SELECT_RELATED.get(model, ())
+        fields = tuple(filter(bool, flatten(rels)))
+        qs = model.objects
+        qs = qs.select_related(*select_related)
+        qs = qs.filter(fk_filter).distinct().values_list(*fields)
+        extra_index = get_key_index(model)
+        has_id = bool(rels[1])
+        existing_mds = {}
+        for existing_values in qs:
+            key = existing_values[:extra_index]
+            value = existing_values[extra_index:]
+            if has_id:
+                identifier = value[:3] if any(value[:3]) else None
+                value = (identifier, *value[3:])
+            existing_mds[key] = value
+        return existing_mds
+
+    def _query_missing_models_batch(  # noqa: PLR0913
+        self,
+        model: type[BaseModel],
+        start: int,
+        all_proposed_key_values: tuple,
+        proposed_values_map: dict[tuple, set[tuple]],
+        create_values: set[tuple],
+        update_values: set[tuple],
+        fts_values: dict[tuple, tuple],
+        status: Status,
+    ):
+        # Do this in batches so as not to exceed the 1k sqlite query depth limit
+        end = start + FILTER_BATCH_SIZE
+        batch_proposed_key_tuples = all_proposed_key_values[start:end]
+        num_in_batch = len(batch_proposed_key_tuples)
+
+        existing_values_map = self.query_existing_mds(model, batch_proposed_key_tuples)
+
+        for key_values in batch_proposed_key_tuples:
+            proposed_extra_values_set = proposed_values_map.pop(key_values)
+            exists = key_values in existing_values_map
+            existing_extra_values = existing_values_map.pop(key_values, None)
+
+            do_update, best_extra_values = self.query_model_best_extra_values(
+                model, existing_extra_values, proposed_extra_values_set
+            )
+            best_values = (*key_values, *best_extra_values)
+            if exists:
+                if do_update:
+                    update_values.add(best_values)
+            else:
+                create_values.add(best_values)
+            if model is Universe:
+                fts_values[key_values] = best_extra_values
+
+        status.increment_complete(num_in_batch)
+        self.status_controller.update(status)
+
+    def _finish_query_missing(
+        self,
+        model: type[BaseModel],
+        values: set | frozenset | dict,
+        key: str,
+        title: str,
+    ):
+        if values:
+            fks = self.metadata[key]
+            if isinstance(values, dict):
+                if model not in fks:
+                    fks[model] = {}
+                fks[model].update(values)
+            else:
+                if model not in fks:
+                    fks[model] = set()
+                fks[model] |= values
+            level = "INFO"
+        else:
+            level = "DEBUG"
+        num = len(values)
+        match key:
+            case FK_KEYS.CREATE_FKS:
+                verb = "create"
+            case FK_KEYS.UPDATE_FKS:
+                verb = "update"
+            case _:
+                verb = ""
+        if verb:
+            self.log.log(level, f"Prepared {num} {title} for {verb}.")
+
+    def query_missing_models(
+        self,
+        model: type[BaseModel],
+        proposed_values_map: dict[tuple, set[tuple]],
+        status: Status,
+    ):
+        """Find missing foreign key models."""
+        if not proposed_values_map:
+            return 0
+        num_all_proposed_values = len(proposed_values_map)
+        proposed_key_values = tuple(proposed_values_map.keys())
+        start = 0
+
+        vnp = model._meta.verbose_name_plural
+        title = vnp.title() if vnp else ""
+        status.subtitle = title
+
+        create_values = set()
+        update_values = set()
+        fts_values = {}
+
+        while start < num_all_proposed_values:
+            self._query_missing_models_batch(
+                model,
+                start,
+                proposed_key_values,
+                proposed_values_map,
+                create_values,
+                update_values,
+                fts_values,
+                status,
+            )
+            start += FILTER_BATCH_SIZE
+
+        self._finish_query_missing(model, create_values, CREATE_FKS, title)
+        self._finish_query_missing(model, update_values, UPDATE_FKS, title)
+        status.subtitle = ""
+        return num_all_proposed_values
+
+    def _query_missing_model(self, model: type[BaseModel], status: Status):
+        """Find missing model and update create and update sets."""
+        count = 0
+        proposed_values = self.metadata[QUERY_MODELS].pop(model, None)
+        if not proposed_values:
+            return count
+
+        # Finally run the query and get only the correct create_objs
+        return self.query_missing_models(
+            model,
+            proposed_values,
+            status,
+        )
+
+    def _set_fk_totals(self, fk_key: str, status_class):
+        total_fks = 0
+        fks = self.metadata[fk_key]
+        for rows in fks.values():
+            total_fks += len(rows)
+        status = status_class(None, total_fks)
+        self.status_controller.update(status, notify=False)
+        self.metadata[fk_key][TOTAL] = total_fks
+
+    def query_all_missing_models(self):
+        """Find all missing foreign key models."""
+        num_models = self.sum_ops(QUERY_MODELS)
+        status = ImporterQueryMissingTagsStatus(0, num_models)
+        try:
+            if not num_models:
+                return num_models
+            self.status_controller.start(status)
+            for model in tuple(self.metadata[QUERY_MODELS].keys()):
+                if self.abort_event.is_set():
+                    return num_models
+                self._query_missing_model(model, status)
+            self.metadata.pop(QUERY_MODELS)
+            self._set_fk_totals(CREATE_FKS, ImporterCreateTagsStatus)
+            self._set_fk_totals(UPDATE_FKS, ImporterUpdateTagsStatus)
+        finally:
+            self.status_controller.finish(status)
+        return status.complete

codex/librarian/scribe/importer/query/update_comics.py

@@ -0,0 +1,59 @@
+"""Move comics that need only updating into correct structure."""
+
+from codex.librarian.scribe.importer.const import (
+    BULK_UPDATE_COMIC_FIELDS,
+    CREATE_COMICS,
+    UPDATE_COMICS,
+)
+from codex.librarian.scribe.importer.query.query_fks import (
+    QueryForeignKeysQueryImporter,
+)
+from codex.librarian.scribe.importer.statii.create import (
+    ImporterCreateComicsStatus,
+    ImporterUpdateComicsStatus,
+)
+from codex.librarian.scribe.importer.statii.query import ImporterQueryComicUpdatesStatus
+from codex.models import Comic
+
+
+class QueryUpdateComics(QueryForeignKeysQueryImporter):
+    """Move comics that need only updating into correct structure."""
+
+    def query_update_comics(self):
+        """Pop existing comics from create & move to update if needed."""
+        paths = tuple(self.metadata[CREATE_COMICS].keys())
+        status = ImporterQueryComicUpdatesStatus(0, len(paths))
+        try:
+            if not paths:
+                return
+            self.status_controller.start(status)
+
+            comics = Comic.objects.filter(library=self.library, path__in=paths).only(
+                *BULK_UPDATE_COMIC_FIELDS
+            )
+            status.increment_complete(len(paths) - comics.count())
+
+            for comic in comics:
+                if self.abort_event.is_set():
+                    return
+                proposed_comic_dict = self.metadata[CREATE_COMICS].pop(comic.path)
+                update_comic_dict = {
+                    key: value
+                    for key, value in proposed_comic_dict.items()
+                    if getattr(comic, key) != value
+                }
+                if update_comic_dict:
+                    self.metadata[UPDATE_COMICS][comic.pk] = update_comic_dict
+                status.increment_complete()
+                self.status_controller.update(status)
+
+            create_status = ImporterCreateComicsStatus(
+                None, len(self.metadata[CREATE_COMICS])
+            )
+            self.status_controller.update(create_status)
+            update_status = ImporterUpdateComicsStatus(
+                None, len(self.metadata[UPDATE_COMICS])
+            )
+            self.status_controller.update(update_status)
+        finally:
+            self.status_controller.finish(status)

codex/librarian/scribe/importer/query/update_fks.py

@@ -0,0 +1,140 @@
+"""Query the missing foreign keys methods."""
+
+from comicbox.enums.comicbox import compare_identifier_source
+
+from codex.librarian.scribe.importer.const import MODEL_REL_MAP
+from codex.librarian.scribe.importer.query.filters import (
+    QueryForeignKeysFilterImporter,
+)
+from codex.models.base import BaseModel
+
+
+class QueryIsUpdateImporter(QueryForeignKeysFilterImporter):
+    """Query the missing foreign keys methods."""
+
+    @staticmethod
+    def _query_missing_models_is_do_update_identifier(
+        best_extra_values: tuple,
+        proposed_extra_values: tuple,
+        synthesized_extra_proposed_values: list,
+    ):
+        """Compare Identifiers."""
+        best_identifier = best_extra_values[0]
+        if proposed_identifier := proposed_extra_values[0]:
+            if best_identifier:
+                best_id_source, _, best_id_key = best_identifier
+                (
+                    proposed_id_source,
+                    _,
+                    proposed_id_key,
+                ) = proposed_identifier
+
+                if (
+                    compare_identifier_source(best_id_source, proposed_id_source)
+                    or best_id_key != proposed_id_key
+                ):
+                    best_identifier = proposed_identifier
+            else:
+                best_identifier = proposed_identifier
+
+        synthesized_extra_proposed_values.append(best_identifier)
+
+    @staticmethod
+    def _query_missing_models_is_do_update_extra(
+        best_extra_values: tuple,
+        proposed_extra_values: tuple,
+        synthesized_extra_proposed_values: list,
+        *,
+        has_id: bool,
+    ):
+        """Compare extra relations."""
+        if has_id:
+            best_extra_values = best_extra_values[1:]
+            proposed_extra_values = proposed_extra_values[1:]
+        synthesized_extra_values = []
+
+        for best_extra_value, proposed_extra_value in zip(
+            best_extra_values, proposed_extra_values, strict=True
+        ):
+            if (
+                proposed_extra_value is not None
+                and proposed_extra_value != best_extra_value
+            ):
+                synthesized_extra_values.append(proposed_extra_value)
+            else:
+                synthesized_extra_values.append(best_extra_value)
+
+        synthesized_extra_proposed_values.extend(synthesized_extra_values)
+
+    @staticmethod
+    def _query_normalize_existing_values(
+        existing_values_tuple: tuple, extra_index: int
+    ):
+        existing_identifier = existing_values_tuple[extra_index : extra_index + 3]
+        if not any(existing_identifier):
+            existing_identifier = None
+        existing_extra_values = existing_values_tuple[extra_index + 3 :]
+        return (existing_identifier, *existing_extra_values)
+
+    @classmethod
+    def _query_update_init_best_and_existing_values(
+        cls,
+        existing_extra_values_tuple: tuple | None,
+        proposed_extra_values_tuples_set: set[tuple],
+    ) -> tuple[tuple | None, tuple]:
+        """
+        Initialize first best value.
+
+        Normalize existing values and assign to best value OR initialize best values
+        with first proposed value.
+        """
+        if existing_extra_values_tuple:
+            best_extra_values = existing_extra_values_tuple
+        else:
+            try:
+                best_extra_values = proposed_extra_values_tuples_set.pop()
+            except KeyError:
+                best_extra_values = ()
+        return existing_extra_values_tuple, best_extra_values
+
+    @classmethod
+    def query_model_best_extra_values(
+        cls,
+        model: type[BaseModel],
+        existing_extra_values_tuple: tuple | None,
+        proposed_extra_values_tuples_set: set[tuple],
+    ) -> tuple[bool, tuple]:
+        """Find possible updates from existing."""
+        if not existing_extra_values_tuple and not proposed_extra_values_tuples_set:
+            return False, ()
+        _, id_rel, *extra_rels = MODEL_REL_MAP[model]
+        existing_extra_values, best_extra_values = (
+            cls._query_update_init_best_and_existing_values(
+                existing_extra_values_tuple,
+                proposed_extra_values_tuples_set,
+            )
+        )
+
+        has_id = bool(id_rel)
+        for extra_values_tuple in proposed_extra_values_tuples_set:
+            synthesized_extra_proposed_values = []
+            if has_id:
+                cls._query_missing_models_is_do_update_identifier(
+                    best_extra_values,
+                    extra_values_tuple,
+                    synthesized_extra_proposed_values,
+                )
+
+            if extra_rels:
+                cls._query_missing_models_is_do_update_extra(
+                    best_extra_values,
+                    extra_values_tuple,
+                    synthesized_extra_proposed_values,
+                    has_id=has_id,
+                )
+
+            best_extra_values = tuple(synthesized_extra_proposed_values)
+
+        do_update = best_extra_values != existing_extra_values
+
+        return do_update, best_extra_values

codex/librarian/scribe/importer/read/__init__.py

@@ -0,0 +1,14 @@
+"""Extract and Aggregate metadata from comic archive."""
+
+from codex.librarian.scribe.importer.read.extract import (
+    ExtractMetadataImporter,
+)
+
+
+class ReadMetadataImporter(ExtractMetadataImporter):
+    """Extract and Aggregate metadata from comics to prepare for importing."""
+
+    def read(self):
+        """Extract and aggregate metadata."""
+        self.extract_metadata()
+        self.aggregate_metadata()

codex/librarian/scribe/importer/read/aggregate_path.py

@@ -0,0 +1,108 @@
+"""Aggregate metadata from comics to prepare for importing."""
+
+from comicbox.schemas.comicbox import (
+    COVER_DATE_KEY,
+    DATE_KEY,
+    NUMBER_KEY,
+    STORE_DATE_KEY,
+    SUFFIX_KEY,
+    TITLE_KEY,
+)
+
+from codex.librarian.scribe.importer.const import (
+    CREATE_COMICS,
+    EXTRACTED,
+    FIS,
+    LINK_FKS,
+    LINK_M2MS,
+    QUERY_MODELS,
+)
+from codex.librarian.scribe.importer.read.folders import AggregatePathMetadataImporter
+from codex.librarian.scribe.importer.statii.read import ImporterAggregateStatus
+from codex.librarian.scribe.importer.status import ImporterFailedImportsStatus
+
+_UNUSED_COMICBOX_FIELDS = (
+    "alternate_images",
+    "bookmark",
+    "credit_primaries",
+    "ext",
+    "manga",
+    "pages",
+    "prices",
+    "remainders",
+    "reprints",
+    "updated_at",
+)
+
+
+class AggregateMetadataImporter(AggregatePathMetadataImporter):
+    """Aggregate metadata from comics to prepare for importing."""
+
+    @staticmethod
+    def _transform_metadata(md):
+        for key in _UNUSED_COMICBOX_FIELDS:
+            md.pop(key, None)
+
+        if date := md.pop(DATE_KEY, None):
+            date.pop(COVER_DATE_KEY, None)
+            date.pop(STORE_DATE_KEY, None)
+            md.update(date)
+
+        if issue := md.pop("issue", None):
+            if number := issue.pop(NUMBER_KEY, None):
+                md["issue_number"] = number
+            if suffix := issue.pop(SUFFIX_KEY, None):
+                md["issue_suffix"] = suffix
+
+        if title := md.pop(TITLE_KEY, None):
+            md["name"] = title
+
+    def _aggregate_path(self, path, status):
+        """Aggregate metadata for one path."""
+        # Prepare
+        md = self.metadata[EXTRACTED].pop(path)
+        self._transform_metadata(md)
+
+        # Aggregate
+        self.metadata[LINK_FKS][path] = {}
+        self.get_fk_metadata(md, path)
+        self.get_m2m_metadata(md, path)
+        if md:
+            self.get_path_metadata(md, path)
+        self.metadata[CREATE_COMICS][str(path)] = md
+
+        # Status
+        status.increment_complete()
+        self.status_controller.update(status)
+
+    def aggregate_metadata(
+        self,
+    ):
+        """Get aggregated metadata for the paths given."""
+        num_extracted_paths = len(self.metadata[EXTRACTED])
+        self.log.debug(
+            f"Aggregating tags from {num_extracted_paths} comics in {self.library.path}..."
+        )
+        status = ImporterAggregateStatus(0, num_extracted_paths)
+        self.status_controller.start(status)
+
+        # Init metadata, extract and aggregate
+        self.metadata[QUERY_MODELS] = {}
+        self.metadata[CREATE_COMICS] = {}
+        self.metadata[LINK_FKS] = {}
+        self.metadata[LINK_M2MS] = {}
+        # Aggregate further
+
+        for path in tuple(self.metadata[EXTRACTED]):
+            if self.abort_event.is_set():
+                return status.complete
+            self._aggregate_path(path, status)
+        del self.metadata[EXTRACTED]
+
+        fis = self.metadata[FIS].keys()
+
+        # Set statii
+        fi_status = ImporterFailedImportsStatus(0, len(fis))
+        self.status_controller.update(fi_status, notify=False)
+        self.status_controller.finish(status)
+        return status.complete

codex/librarian/scribe/importer/read/const.py

@@ -0,0 +1,144 @@
+"""Aggregate Consts."""
+
+from types import MappingProxyType
+
+from comicbox.schemas.comicbox import (
+    ARCS_KEY,
+    CHARACTERS_KEY,
+    DESIGNATION_KEY,
+    GENRES_KEY,
+    ID_KEY_KEY,
+    ID_URL_KEY,
+    IDENTIFIERS_KEY,
+    LOCATIONS_KEY,
+    NUMBER_KEY,
+    PROTAGONIST_KEY,
+    ROLES_KEY,
+    SERIES_GROUPS_KEY,
+    STORIES_KEY,
+    TAGS_KEY,
+    TEAMS_KEY,
+)
+from django.db.models.fields import CharField, Field
+from django.db.models.fields.related_descriptors import ForwardManyToOneDescriptor
+from django.db.models.query_utils import DeferredAttribute
+
+from codex.librarian.scribe.importer.const import (
+    ALL_COMIC_FK_FIELDS,
+    CREDITS_FIELD_NAME,
+    IDENTIFIERS_FIELD_NAME,
+    NAME_FIELD_NAME,
+    PROTAGONIST_FIELD_MODEL_MAP,
+    STORY_ARC_NUMBERS_FIELD_NAME,
+    UNIVERSES_FIELD_NAME,
+)
+from codex.models import (
+    StoryArc,
+    StoryArcNumber,
+    Universe,
+)
+from codex.models.identifier import (
+    Identifier,
+    IdentifierSource,
+)
+from codex.models.named import (
+    Character,
+    CreditPerson,
+    CreditRole,
+    Genre,
+    Location,
+    SeriesGroup,
+    Story,
+    Tag,
+    Team,
+)
+
+######
+# FK #
+######
+_EXCLUDE_FIELD_NAMES = frozenset(PROTAGONIST_FIELD_MODEL_MAP.keys() | {"parent_folder"})
+COMIC_FK_FIELD_NAMES_FIELD_MAP: MappingProxyType[str, Field] = MappingProxyType(
+    {
+        **{
+            field.name: field.related_model._meta.get_field(NAME_FIELD_NAME)
+            for field in ALL_COMIC_FK_FIELDS
+            if field.related_model and field.name not in _EXCLUDE_FIELD_NAMES
+        },
+        PROTAGONIST_KEY: PROTAGONIST_FIELD_MODEL_MAP["main_character"]._meta.get_field(
+            NAME_FIELD_NAME
+        ),
+    }
+)
+
+#######
+# M2M #
+#######
+FIELD_NAME_TO_MD_KEY_MAP = MappingProxyType(
+    {
+        STORY_ARC_NUMBERS_FIELD_NAME: ARCS_KEY,
+    }
+)
+SIMPLE_KEY_CLASS_MAP = MappingProxyType(
+    {
+        SERIES_GROUPS_KEY: SeriesGroup,
+    }
+)
+IDENTIFIED_KEY_CLASS_MAP = MappingProxyType(
+    {
+        CHARACTERS_KEY: Character,
+        GENRES_KEY: Genre,
+        LOCATIONS_KEY: Location,
+        STORIES_KEY: Story,
+        TAGS_KEY: Tag,
+        TEAMS_KEY: Team,
+    }
+)
+ID_TYPE_KEY = "id_type"
+# This map tells aggregator how to parse metadata into tuples for query & create.
+COMPLEX_FIELD_AGG_MAP: MappingProxyType[
+    str,
+    tuple[
+        DeferredAttribute | CharField,
+        ForwardManyToOneDescriptor | None,
+        dict[str, DeferredAttribute],
+    ],
+] = MappingProxyType(
+    {
+        **{key: (cls.name, None, {}) for key, cls in SIMPLE_KEY_CLASS_MAP.items()},
+        **{
+            key: (cls.name, cls.identifier, {})
+            for key, cls in IDENTIFIED_KEY_CLASS_MAP.items()
+        },
+        CREDITS_FIELD_NAME: (
+            CreditPerson.name,
+            CreditPerson.identifier,
+            {
+                ROLES_KEY: CreditRole.name,
+            },
+        ),
+        IDENTIFIERS_FIELD_NAME: (
+            IdentifierSource.name,
+            None,
+            {
+                ID_TYPE_KEY: "comic",
+                ID_KEY_KEY: Identifier.key,
+                ID_URL_KEY: Identifier.url,
+            },
+        ),
+        STORY_ARC_NUMBERS_FIELD_NAME: (
+            StoryArc.name,
+            StoryArc.identifier,
+            {
+                NUMBER_KEY: StoryArcNumber.number,
+            },
+        ),
+        UNIVERSES_FIELD_NAME: (
+            Universe.name,
+            None,
+            {
+                IDENTIFIERS_KEY: Universe.identifier,
+                DESIGNATION_KEY: Universe.designation,
+            },
+        ),
+    }
+)