codex 1.7.14__py3-none-any.whl → 1.8.0__py3-none-any.whl

codex/librarian/importer/const.py

@@ -1,244 +0,0 @@
-"""Consts and maps for import."""
-
-from types import MappingProxyType
-
-from bidict import bidict
-
-from codex.models import (
-    Comic,
-    Contributor,
-    Folder,
-    Imprint,
-    Publisher,
-    Series,
-    StoryArc,
-    StoryArcNumber,
-    Volume,
-)
-from codex.models.groups import BrowserGroupModel
-from codex.models.named import Identifier
-from codex.models.paths import CustomCover
-
-#################
-# DICT METADATA #
-#################
-CONTRIBUTORS_FIELD_NAME = "contributors"
-_CONTRIBUTOR_PERSON_FIELD_NAME = "person"
-_CONTRIBUTOR_ROLE_FIELD_NAME = "role"
-STORY_ARCS_METADATA_KEY = "story_arcs"
-STORY_ARC_NUMBERS_FK_NAME = "story_arc_numbers"
-_STORY_ARC_FIELD_NAME = "story_arc"
-_NUMBER_FIELD_NAME = "number"
-IDENTIFIERS_FIELD_NAME = "identifiers"
-_IDENTIFIER_TYPE_FIELD_NAME = "identifier_type"
-_IDENTIFIER_CODE_FIELD_NAME = "nss"
-
-# AGGREGATE
-DICT_MODEL_AGG_MAP = MappingProxyType(
-    {
-        CONTRIBUTORS_FIELD_NAME: (
-            _CONTRIBUTOR_ROLE_FIELD_NAME,
-            _CONTRIBUTOR_PERSON_FIELD_NAME,
-        ),
-        STORY_ARCS_METADATA_KEY: (_STORY_ARC_FIELD_NAME, None),
-        IDENTIFIERS_FIELD_NAME: (_IDENTIFIER_TYPE_FIELD_NAME, None),
-    }
-)
-## QUERY
-DICT_MODEL_CLASS_FIELDS_MAP = MappingProxyType(
-    {
-        Contributor: frozenset(
-            {_CONTRIBUTOR_ROLE_FIELD_NAME, _CONTRIBUTOR_PERSON_FIELD_NAME}
-        ),
-        StoryArcNumber: frozenset({_STORY_ARC_FIELD_NAME}),
-        Identifier: frozenset({_IDENTIFIER_TYPE_FIELD_NAME}),
-    }
-)
-DICT_MODEL_FIELD_MODEL_MAP = MappingProxyType(
-    {
-        CONTRIBUTORS_FIELD_NAME: Contributor,
-        STORY_ARCS_METADATA_KEY: StoryArcNumber,
-        IDENTIFIERS_FIELD_NAME: Identifier,
-    }
-)
-DICT_MODEL_REL_MAP = MappingProxyType(
-    {
-        CONTRIBUTORS_FIELD_NAME: (
-            f"{_CONTRIBUTOR_ROLE_FIELD_NAME}__name",
-            f"{_CONTRIBUTOR_PERSON_FIELD_NAME}__name",
-        ),
-        STORY_ARCS_METADATA_KEY: (
-            f"{_STORY_ARC_FIELD_NAME}__name",
-            _NUMBER_FIELD_NAME,
-        ),
-        IDENTIFIERS_FIELD_NAME: (
-            f"{_IDENTIFIER_TYPE_FIELD_NAME}__name",
-            {
-                "nss": _IDENTIFIER_CODE_FIELD_NAME,
-            },
-        ),
-    }
-)
-# CREATE
-CREATE_DICT_UPDATE_FIELDS = MappingProxyType(
-    {
-        Contributor: (_CONTRIBUTOR_ROLE_FIELD_NAME, _CONTRIBUTOR_PERSON_FIELD_NAME),
-        StoryArcNumber: (_STORY_ARC_FIELD_NAME, _NUMBER_FIELD_NAME),
-        Identifier: (_IDENTIFIER_TYPE_FIELD_NAME, _IDENTIFIER_CODE_FIELD_NAME),
-    }
-)
-# LINK
-DICT_MODEL_FIELD_NAME_CLASS_MAP = (
-    (CONTRIBUTORS_FIELD_NAME, Contributor),
-    (STORY_ARCS_METADATA_KEY, StoryArcNumber),
-    (IDENTIFIERS_FIELD_NAME, Identifier),
-)
-DICT_MODEL_REL_LINK_MAP = MappingProxyType(
-    {
-        CONTRIBUTORS_FIELD_NAME: (
-            f"{_CONTRIBUTOR_ROLE_FIELD_NAME}__name",
-            f"{_CONTRIBUTOR_PERSON_FIELD_NAME}__name__in",
-        ),
-        STORY_ARC_NUMBERS_FK_NAME: (
-            f"{_STORY_ARC_FIELD_NAME}__name",
-            _NUMBER_FIELD_NAME,
-        ),
-        IDENTIFIERS_FIELD_NAME: (
-            f"{_IDENTIFIER_TYPE_FIELD_NAME}__name",
-            _IDENTIFIER_CODE_FIELD_NAME,
-        ),
-    }
-)
-
-
-###############
-# BULK UPDATE #
-###############
-
-_EXCLUDEBULK_UPDATE_COMIC_FIELDS = {
-    "bookmark",
-    "created_at",
-    "id",
-    "library",
-    "comicfts",
-}
-GROUP_BASE_FIELDS = ("name", "sort_name")
-BULK_UPDATE_COMIC_FIELDS = tuple(
-    sorted(
-        field.name
-        for field in Comic._meta.get_fields()
-        if (not field.many_to_many)
-        and (field.name not in _EXCLUDEBULK_UPDATE_COMIC_FIELDS)
-    )
-)
-BULK_CREATE_COMIC_FIELDS = (*BULK_UPDATE_COMIC_FIELDS, "library")
-BULK_UPDATE_FOLDER_FIELDS = (
-    *GROUP_BASE_FIELDS,
-    "stat",
-    "updated_at",
-    "path",
-    "parent_folder",
-)
-BULK_UPDATE_FOLDER_MODIFIED_FIELDS = ("stat", "updated_at")
-BULK_UPDATE_COMIC_FIELDS_WITH_VALUES = tuple(
-    sorted(
-        frozenset(BULK_UPDATE_COMIC_FIELDS)
-        - frozenset(BULK_UPDATE_FOLDER_MODIFIED_FIELDS)
-    )
-)
-MOVED_BULK_COMIC_UPDATE_FIELDS = ("path", "parent_folder", "stat", "updated_at")
-MOVED_BULK_FOLDER_UPDATE_FIELDS = (
-    "path",
-    "parent_folder",
-    *GROUP_BASE_FIELDS,
-    "stat",
-    "updated_at",
-)
-CUSTOM_COVER_UPDATE_FIELDS = ("path", "stat", "updated_at", "sort_name", "group")
-
-#########
-# OTHER #
-#########
-VOLUME_COUNT = "volume_count"
-ISSUE_COUNT = "issue_count"
-COUNT_FIELDS = {Series: VOLUME_COUNT, Volume: ISSUE_COUNT}
-_GROUP_CLASSES = (Publisher, Imprint, Series, Volume)
-
-
-def _create_group_update_fields():
-    guf = {}
-    fields = GROUP_BASE_FIELDS
-    for cls in _GROUP_CLASSES:
-        if cls == Volume:
-            guf[cls] = tuple({*fields} - {"sort_name"})
-        else:
-            guf[cls] = fields
-        fields = (*fields, cls.__name__.lower())
-    return MappingProxyType(guf)
-
-
-GROUP_UPDATE_FIELDS = _create_group_update_fields()
-NAMED_MODEL_UPDATE_FIELDS = ("name",)
-FOLDERS_FIELD = "folders"
-GROUP_TREES = "group_trees"
-COMIC_PATHS = "comic_paths"
-PUBLISHER = "publisher"
-IMPRINT = "imprint"
-VOLUME = "volume"
-SERIES = "series"
-PARENT_FOLDER = "parent_folder"
-MDS = "mds"
-M2M_MDS = "m2m_mds"
-FKS = "fks"
-FIS = "fis"
-FK_CREATE = "fk_create"
-COVERS_UPDATE = "covers_update"
-COVERS_CREATE = "covers_create"
-LINK_COVER_PKS = "link_cover_pks"
-GROUP_COMPARE_FIELDS = MappingProxyType(
-    {
-        Series: ("publisher__name", "imprint__name", "name"),
-        Volume: ("publisher__name", "imprint__name", "series__name", "name"),
-    }
-)
-CLASS_CUSTOM_COVER_GROUP_MAP = bidict(
-    {
-        Publisher: CustomCover.GroupChoice.P.value,
-        Imprint: CustomCover.GroupChoice.I.value,
-        Series: CustomCover.GroupChoice.S.value,
-        StoryArc: CustomCover.GroupChoice.A.value,
-        Folder: CustomCover.GroupChoice.F.value,
-    }
-)
-
-COMIC_FK_FIELD_NAMES = frozenset(
-    field.name
-    for field in Comic._meta.get_fields()
-    if field.many_to_one
-    and field.name != "library"
-    and field.related_model
-    and not issubclass(field.related_model, BrowserGroupModel)
-)
-COMIC_M2M_FIELD_NAMES = frozenset(
-    # Leaves out folders.
-    field.name
-    for field in Comic._meta.get_fields()
-    if field.many_to_many and field.name != "folders"
-)
-COMIC_GROUP_FIELD_NAMES = (
-    "publisher",
-    "imprint",
-    "series",
-    "volume",
-    "story_arc_numbers",
-    "folders",
-)
-
-FKC_CONTRIBUTORS = "create_contributors"
-FKC_STORY_ARC_NUMBERS = "create_story_arc_numbers"
-FKC_IDENTIFIERS = "create_identifiers"
-FKC_CREATE_GROUPS = "create_groups"
-FKC_UPDATE_GROUPS = "update_groups"
-FKC_CREATE_FKS = "create_fks"
-FKC_FOLDER_PATHS = "create_folder_paths"
-FKC_TOTAL_FKS = "total_fks"

codex/librarian/importer/create_comics.py

@@ -1,139 +0,0 @@
-"""Bulk update and create comic objects and bulk update m2m fields."""
-
-from django.db.models import NOT_PROVIDED
-from django.db.models.functions import Now
-
-from codex.librarian.importer.const import (
-    BULK_CREATE_COMIC_FIELDS,
-    BULK_UPDATE_COMIC_FIELDS,
-    BULK_UPDATE_COMIC_FIELDS_WITH_VALUES,
-    MDS,
-)
-from codex.librarian.importer.link_comics import LinkComicsImporter
-from codex.librarian.importer.status import ImportStatusTypes
-from codex.models import (
-    Comic,
-)
-from codex.status import Status
-
-
-class CreateComicsImporter(LinkComicsImporter):
-    """Create comics methods."""
-
-    def _bulk_update_comics_add_comic(self, comic, results):
-        """Add one comic and stats to the bulk update list."""
-        try:
-            md = self.metadata[MDS].pop(comic.path)
-            self.get_comic_fk_links(md, comic.path)
-            for field_name in BULK_UPDATE_COMIC_FIELDS_WITH_VALUES:
-                value = md.get(field_name)
-                if value is None:
-                    default_value = Comic._meta.get_field(field_name).default
-                    if default_value != NOT_PROVIDED:
-                        value = default_value
-                setattr(comic, field_name, value)
-            comic.presave()
-            comic.updated_at = Now()
-            update_comics, comic_pks, comic_update_paths = results
-            update_comics.append(comic)
-            comic_pks.append(comic.pk)
-            comic_update_paths.add(comic.path)
-        except Exception:
-            self.log.exception(f"Error preparing {comic} for update.")
-
-    def bulk_update_comics(self):
-        """Bulk update comics, and move nonextant comics into create job.."""
-        num_comics = len(self.task.files_modified)
-        if not num_comics:
-            return num_comics
-
-        self.log.debug(
-            f"Preparing {num_comics} comics for update in library {self.library.path}."
-        )
-        status = Status(ImportStatusTypes.FILES_MODIFIED, None, num_comics)
-        self.status_controller.start(status)
-        # Get existing comics to update
-        comics = Comic.objects.filter(
-            library=self.library, path__in=self.task.files_modified
-        ).only(*BULK_UPDATE_COMIC_FIELDS)
-
-        # set attributes for each comic
-        update_comics = []
-        comic_pks = []
-        comic_update_paths = set()
-        results = update_comics, comic_pks, comic_update_paths
-        for comic in comics.iterator():
-            self._bulk_update_comics_add_comic(comic, results)
-
-        converted_create_paths = frozenset(
-            set(self.task.files_modified) - comic_update_paths
-        )
-        self.task.files_created |= converted_create_paths
-        count = len(converted_create_paths)
-        if count:
-            self.log.info(f"Converted {count} update paths to create paths.")
-
-        self.log.debug(f"Bulk updating {len(update_comics)} comics.")
-        try:
-            Comic.objects.bulk_update(update_comics, BULK_UPDATE_COMIC_FIELDS)
-            count = len(update_comics)
-
-            self._remove_covers(comic_pks, custom=False)
-            self.log.debug(f"Purging covers for {len(comic_pks)} updated comics.")
-            if count:
-                self.log.info(f"Updated {count} comics.")
-        except Exception:
-            self.log.exception(f"While updating {comic_update_paths}")
-
-        self.task.files_modified = frozenset()
-
-        self.status_controller.finish(status)
-        return count
-
-    def bulk_create_comics(self):
-        """Bulk create comics."""
-        num_comics = len(self.task.files_created)
-        if not num_comics:
-            return num_comics
-        # prepare create comics
-        self.log.debug(
-            f"Preparing {num_comics} comics for creation in library {self.library.path}."
-        )
-        status = Status(ImportStatusTypes.FILES_CREATED, None, num_comics)
-        self.status_controller.start(status)
-
-        create_comics = []
-        for path in sorted(self.task.files_created):
-            try:
-                md = self.metadata[MDS].pop(path, {})
-                self.get_comic_fk_links(md, path)
-                comic = Comic(**md, library=self.library)
-                comic.presave()
-                create_comics.append(comic)
-            except KeyError:
-                self.log.warning(f"No comic metadata for {path}")
-            except Exception:
-                self.log.exception(f"Error preparing {path} for create.")
-        self.task.files_created = frozenset()
-        self.metadata.pop(MDS)
-
-        num_comics = len(create_comics)
-        count = 0
-        if num_comics:
-            self.log.debug(f"Bulk creating {num_comics} comics...")
-            try:
-                Comic.objects.bulk_create(
-                    create_comics,
-                    update_conflicts=True,
-                    update_fields=BULK_CREATE_COMIC_FIELDS,
-                    unique_fields=Comic._meta.unique_together[0],
-                )
-                count = len(create_comics)
-                if count:
-                    self.log.info(f"Created {count} comics.")
-            except Exception:
-                self.log.exception(f"While creating {num_comics} comics")
-
-        self.changed += count
-        self.status_controller.finish(status)
-        return count

codex/librarian/importer/create_covers.py

@@ -1,96 +0,0 @@
-"""Create Custom Covers."""
-
-from django.db.models.functions.datetime import Now
-
-from codex.librarian.importer.const import (
-    COVERS_CREATE,
-    COVERS_UPDATE,
-    CUSTOM_COVER_UPDATE_FIELDS,
-    LINK_COVER_PKS,
-)
-from codex.librarian.importer.create_comics import CreateComicsImporter
-from codex.librarian.importer.status import ImportStatusTypes
-from codex.models import (
-    CustomCover,
-)
-from codex.status import Status
-
-
-class CreateCoversImporter(CreateComicsImporter):
-    """Create Custom Covers."""
-
-    def update_custom_covers(self, status=None):
-        """Update Custom Covers."""
-        update_covers_qs = self.metadata.pop(COVERS_UPDATE, None)
-        if not update_covers_qs:
-            return
-        update_covers_count = update_covers_qs.count()
-        if not update_covers_count:
-            return
-        status = Status(ImportStatusTypes.COVERS_MODIFIED, 0, update_covers_count)
-        self.status_controller.start(status)
-
-        update_covers = []
-        for cover in update_covers_qs.only(*CUSTOM_COVER_UPDATE_FIELDS):
-            cover.updated_at = Now()
-            cover.presave()
-            update_covers.append(cover)
-
-        count = 0
-        if update_covers:
-            CustomCover.objects.bulk_update(update_covers, CUSTOM_COVER_UPDATE_FIELDS)
-            update_cover_pks = update_covers_qs.values_list("pk", flat=True)
-            if LINK_COVER_PKS not in self.metadata:
-                self.metadata[LINK_COVER_PKS] = set()
-            self.metadata[LINK_COVER_PKS].update(update_cover_pks)
-            self._remove_covers(update_cover_pks, custom=True)
-            count = len(update_covers)
-            if status:
-                status.add_complete(count)
-
-            link_covers_status = Status(
-                ImportStatusTypes.COVERS_LINK, 0, len(self.metadata[LINK_COVER_PKS])
-            )
-            self.status_controller.update(link_covers_status, notify=False)
-
-        self.changed += count
-        self.status_controller.finish(status)
-
-    def create_custom_covers(self, status=None):
-        """Create Custom Covers."""
-        create_cover_paths = self.metadata.pop(COVERS_CREATE, ())
-        num_create_cover_paths = len(create_cover_paths)
-        if not num_create_cover_paths:
-            return
-        status = Status(ImportStatusTypes.COVERS_MODIFIED, 0, num_create_cover_paths)
-        self.status_controller.start(status)
-
-        create_covers = []
-        for path in create_cover_paths:
-            cover = CustomCover(library=self.library, path=path)
-            cover.presave()
-            create_covers.append(cover)
-
-        count = 0
-        if create_covers:
-            objs = CustomCover.objects.bulk_create(
-                create_covers,
-                update_conflicts=True,
-                update_fields=("path", "stat"),
-                unique_fields=CustomCover._meta.unique_together[0],
-            )
-            created_pks = frozenset(obj.pk for obj in objs)
-            if LINK_COVER_PKS not in self.metadata:
-                self.metadata[LINK_COVER_PKS] = set()
-            self.metadata[LINK_COVER_PKS].update(created_pks)
-            count = len(created_pks)
-            if status:
-                status.add_complete(count)
-
-            link_covers_status = Status(
-                ImportStatusTypes.COVERS_LINK, 0, len(self.metadata[LINK_COVER_PKS])
-            )
-            self.status_controller.update(link_covers_status, notify=False)
-
-        self.changed += count
-        self.status_controller.finish(status)