codex 1.7.15__py3-none-any.whl → 1.8.0__py3-none-any.whl

codex/librarian/importer/init.py

@@ -1,264 +0,0 @@
-"""Initiale Importer."""
-
-import logging
-from pathlib import Path
-from time import sleep, time
-from typing import Any
-
-from django.utils.timezone import now
-
-from codex.librarian.importer.status import ImportStatusTypes
-from codex.librarian.importer.tasks import ImportDBDiffTask
-from codex.librarian.search.status import SearchIndexStatusTypes
-from codex.librarian.search.tasks import SearchIndexAbortTask
-from codex.models import Library
-from codex.status import Status
-from codex.worker_base import WorkerBaseMixin
-
-_WRITE_WAIT_EXPIRY = 60
-
-
-class InitImporter(WorkerBaseMixin):
-    """Initiale Importer."""
-
-    def __init__(self, task: ImportDBDiffTask, log_queue, librarian_queue):
-        """Initialize the import."""
-        self.init_worker(log_queue, librarian_queue)
-        self.task: ImportDBDiffTask = task
-        self.metadata: dict[str, Any] = {}
-        self.changed: int = 0
-        self.library = Library.objects.only("path", "update_in_progress").get(
-            pk=self.task.library_id
-        )
-
-    def _wait_for_filesystem_ops_to_finish(self) -> bool:
-        """Watchdog sends events before filesystem events finish, so wait for them."""
-        started_checking = time()
-
-        # Don't wait for deletes to complete.
-        # Do wait for move, modified, create files before import.
-        all_modified_paths = (
-            frozenset(self.task.dirs_moved.values())
-            | frozenset(self.task.files_moved.values())
-            | self.task.dirs_modified
-            | self.task.files_modified
-            | self.task.files_created
-        )
-
-        old_total_size = -1
-        total_size = 0
-        wait_time = 2
-        while old_total_size != total_size:
-            if old_total_size > 0:
-                # second time around or more
-                sleep(wait_time)
-                wait_time = wait_time**2
-                self.log.debug(
-                    f"Waiting for files to copy before import: "
-                    f"{old_total_size} != {total_size}"
-                )
-            if time() - started_checking > _WRITE_WAIT_EXPIRY:
-                return True
-
-            old_total_size = total_size
-            total_size = 0
-            for path_str in all_modified_paths:
-                path = Path(path_str)
-                if path.exists():
-                    total_size += Path(path).stat().st_size
-        return False
-
-    #######
-    # LOG #
-    #######
-    def _log_task_construct_dirs_log(self):
-        """Construct dirs log line."""
-        dirs_log = []
-        if self.task.dirs_moved:
-            dirs_log += [f"{len(self.task.dirs_moved)} moved"]
-        if self.task.dirs_modified:
-            dirs_log += [f"{len(self.task.dirs_modified)} modified"]
-        if self.task.dirs_deleted:
-            dirs_log += [f"{len(self.task.dirs_deleted)} deleted"]
-        return dirs_log
-
-    def _log_task_construct_comics_log(self):
-        """Construct comcis log line."""
-        comics_log = []
-        if self.task.files_moved:
-            comics_log += [f"{len(self.task.files_moved)} moved"]
-        if self.task.files_modified:
-            comics_log += [f"{len(self.task.files_modified)} modified"]
-        if self.task.files_created:
-            comics_log += [f"{len(self.task.files_created)} created"]
-        if self.task.files_deleted:
-            comics_log += [f"{len(self.task.files_deleted)} deleted"]
-        return comics_log
-
-    def _log_task(self):
-        """Log the watchdog event self.task."""
-        if self.log.getEffectiveLevel() < logging.DEBUG:
-            return
-
-        self.log.debug(f"Updating library {self.library.path}...")
-        comics_log = self._log_task_construct_comics_log()
-        if comics_log:
-            log = "Comics: "
-            log += ", ".join(comics_log)
-            self.log.debug("  " + log)
-
-        dirs_log = self._log_task_construct_dirs_log()
-        if dirs_log:
-            log = "Folders: "
-            log += ", ".join(dirs_log)
-            self.log.debug("  " + log)
-
-    ########
-    # INIT #
-    ########
-    def _init_librarian_status_moved(self, status_list):
-        """Initialize moved statuses."""
-        search_index_updates = 0
-        if self.task.dirs_moved:
-            status_list += [
-                Status(ImportStatusTypes.DIRS_MOVED, None, len(self.task.dirs_moved))
-            ]
-        if self.task.files_moved:
-            status_list += [
-                Status(ImportStatusTypes.FILES_MOVED, None, len(self.task.files_moved))
-            ]
-            search_index_updates += len(self.task.files_moved)
-        if self.task.covers_moved:
-            status_list += [
-                Status(
-                    ImportStatusTypes.COVERS_MOVED, None, len(self.task.covers_moved)
-                )
-            ]
-        if self.task.dirs_modified:
-            status_list += [
-                Status(
-                    ImportStatusTypes.DIRS_MODIFIED, None, len(self.task.dirs_modified)
-                )
-            ]
-        return search_index_updates
-
-    def _init_if_modified_or_created(self, path, status_list):
-        """Initialize librarian statuses for modified or created ops."""
-        total_paths = len(self.task.files_modified) + len(self.task.files_created)
-        status_list += [
-            Status(ImportStatusTypes.AGGREGATE_TAGS, 0, total_paths, subtitle=path),
-            Status(ImportStatusTypes.QUERY_MISSING_FKS),
-            Status(ImportStatusTypes.QUERY_MISSING_COVERS),
-            Status(ImportStatusTypes.CREATE_FKS),
-        ]
-        if self.task.files_modified:
-            status_list += [
-                Status(
-                    ImportStatusTypes.FILES_MODIFIED,
-                    None,
-                    len(self.task.files_modified),
-                )
-            ]
-        if self.task.covers_modified:
-            status_list += [
-                Status(
-                    ImportStatusTypes.COVERS_MODIFIED,
-                    None,
-                    len(self.task.covers_modified),
-                )
-            ]
-
-        if self.task.files_created or self.task.covers_created:
-            status_list += [
-                Status(
-                    ImportStatusTypes.FILES_CREATED, None, len(self.task.files_created)
-                )
-            ]
-        if self.task.covers_created:
-            status_list += [
-                Status(
-                    ImportStatusTypes.COVERS_CREATED,
-                    None,
-                    len(self.task.covers_created),
-                )
-            ]
-
-        if self.task.files_modified or self.task.files_created:
-            status_list += [Status(ImportStatusTypes.LINK_M2M_FIELDS)]
-
-        num_covers_linked = (
-            len(self.task.covers_moved)
-            + len(self.task.covers_modified)
-            + len(self.task.covers_created)
-        )
-        if num_covers_linked:
-            status_list += [
-                Status(ImportStatusTypes.COVERS_LINK, None, num_covers_linked)
-            ]
-
-        return total_paths
-
-    def _init_librarian_status_deleted(self, status_list):
-        """Init deleted statuses."""
-        search_index_updates = 0
-        if self.task.files_deleted:
-            status_list += [
-                Status(
-                    ImportStatusTypes.FILES_DELETED, None, len(self.task.files_deleted)
-                )
-            ]
-            search_index_updates += len(self.task.files_deleted)
-        if self.task.covers_deleted:
-            status_list += [
-                Status(
-                    ImportStatusTypes.COVERS_DELETED,
-                    None,
-                    len(self.task.covers_deleted),
-                )
-            ]
-        return search_index_updates
-
-    @staticmethod
-    def _init_librarian_status_search_index(search_index_updates, status_list):
-        """Init search index statuses."""
-        status_list += [
-            Status(
-                SearchIndexStatusTypes.SEARCH_INDEX_UPDATE,
-                total=search_index_updates,
-            ),
-            Status(SearchIndexStatusTypes.SEARCH_INDEX_REMOVE),
-        ]
-
-    def _init_librarian_status(self, path):
-        """Update the librarian status self.tasks."""
-        status_list = []
-        search_index_updates = 0
-        search_index_updates += self._init_librarian_status_moved(status_list)
-        if (
-            self.task.files_modified
-            or self.task.files_created
-            or self.task.covers_modified
-            or self.task.covers_created
-        ):
-            search_index_updates += self._init_if_modified_or_created(path, status_list)
-        search_index_updates += self._init_librarian_status_deleted(status_list)
-        status_list += [Status(ImportStatusTypes.GROUP_UPDATE)]
-        self._init_librarian_status_search_index(search_index_updates, status_list)
-        self.status_controller.start_many(status_list)
-
-    def init_apply(self):
-        """Initialize the library and status flags."""
-        self.start_time = now()
-        self.librarian_queue.put(SearchIndexAbortTask())
-        self.library.update_in_progress = True
-        self.library.save()
-        too_long = self._wait_for_filesystem_ops_to_finish()
-        if too_long:
-            self.log.warning(
-                "Import apply waited for the filesystem to stop changing too long. "
-                "Try polling again once files have finished copying"
-                f" in library: {self.library.path}"
-            )
-            return
-        self._log_task()
-        self._init_librarian_status(self.library.path)

codex/librarian/importer/link_comics.py

@@ -1,270 +0,0 @@
-"""Bulk update m2m fields."""
-
-from collections.abc import Mapping
-from pathlib import Path
-
-from django.db.models import Q
-
-from codex.librarian.importer.const import (
-    COMIC_FK_FIELD_NAMES,
-    DICT_MODEL_FIELD_NAME_CLASS_MAP,
-    DICT_MODEL_REL_LINK_MAP,
-    FOLDERS_FIELD,
-    IMPRINT,
-    M2M_MDS,
-    PARENT_FOLDER,
-    PUBLISHER,
-    SERIES,
-    STORY_ARC_NUMBERS_FK_NAME,
-    STORY_ARCS_METADATA_KEY,
-    VOLUME,
-)
-from codex.librarian.importer.link_covers import LinkCoversImporter
-from codex.librarian.importer.status import ImportStatusTypes
-from codex.models import (
-    Comic,
-    Folder,
-    Imprint,
-    Publisher,
-    Series,
-    Volume,
-)
-from codex.models.named import Identifier
-from codex.status import Status
-
-
-class LinkComicsImporter(LinkCoversImporter):
-    """Link comics methods."""
-
-    @staticmethod
-    def _get_group_name(group_class, md):
-        """Get the name of the browse group."""
-        field_name = group_class.__name__.lower()
-        return md.get(field_name, group_class.DEFAULT_NAME)
-
-    @classmethod
-    def get_comic_fk_links(cls, md, path):
-        """Get links for all foreign keys for creating and updating."""
-        publisher_name = cls._get_group_name(Publisher, md)
-        imprint_name = cls._get_group_name(Imprint, md)
-        series_name = cls._get_group_name(Series, md)
-        volume_name = cls._get_group_name(Volume, md)
-        md[PUBLISHER] = Publisher.objects.get(name=publisher_name)
-        md[IMPRINT] = Imprint.objects.get(
-            publisher__name=publisher_name, name=imprint_name
-        )
-        md[SERIES] = Series.objects.get(
-            publisher__name=publisher_name, imprint__name=imprint_name, name=series_name
-        )
-        md[VOLUME] = Volume.objects.get(
-            publisher__name=publisher_name,
-            imprint__name=imprint_name,
-            series__name=series_name,
-            name=volume_name,
-        )
-        parent_path = Path(path).parent
-        md[PARENT_FOLDER] = Folder.objects.get(path=parent_path)
-        for field_name in COMIC_FK_FIELD_NAMES:
-            name = md.pop(field_name, None)
-            if name and (fk_class := Comic._meta.get_field(field_name).related_model):
-                md[field_name] = fk_class.objects.get(name=name)
-
-    @staticmethod
-    def _get_link_folders_filter(_field_name, folder_paths):
-        """Get the ids of all folders to link."""
-        return Q(path__in=folder_paths)
-
-    @staticmethod
-    def _get_link_identifier_filter(field_name, dict_md):
-        """Get the ids of all dict style objects to link."""
-        dict_filter = Q()
-        key_rel, value_rel = DICT_MODEL_REL_LINK_MAP[field_name]
-        for key, values_dict in dict_md.items():
-            nss = values_dict.get("nss")
-            dict_filter |= Q(**{key_rel: key, value_rel: nss})
-        return dict_filter
-
-    @staticmethod
-    def _get_link_dict_filter(field_name, dict_md):
-        """Get the ids of all dict style objects to link."""
-        dict_filter = Q()
-        key_rel, value_rel = DICT_MODEL_REL_LINK_MAP[field_name]
-        for key, value in dict_md.items():
-            rel_dict = {key_rel: key, value_rel: value}
-            dict_filter |= Q(**rel_dict)
-        return dict_filter
-
-    def _link_named_m2ms(self, all_m2m_links, comic_pk, md):
-        """Set the ids of all named m2m fields into the comic dict."""
-        total = 0
-        for field, names in md.items():
-            related_model = Comic._meta.get_field(field).related_model
-            if related_model is None:
-                self.log.error(f"No related class found for Comic.{field}")
-                continue
-            pks = related_model.objects.filter(name__in=names).values_list(
-                "pk", flat=True
-            )
-            if field not in all_m2m_links:
-                all_m2m_links[field] = {}
-            all_m2m_links[field][comic_pk] = frozenset(pks)
-            total += len(pks)
-        return total
-
-    def _link_prepare_special_m2ms(self, link_data, key, model, get_link_filter_method):
-        """Prepare special m2m for linking."""
-        (all_m2m_links, md, comic_pk) = link_data
-        values = md.pop(key, [])
-        if not values:
-            return
-        if key == STORY_ARCS_METADATA_KEY:
-            key = STORY_ARC_NUMBERS_FK_NAME
-        if key not in all_m2m_links:
-            all_m2m_links[key] = {}
-
-        m2m_filter = get_link_filter_method(key, values)
-        pks = model.objects.filter(m2m_filter).values_list("pk", flat=True).distinct()
-        result = frozenset(pks)
-        all_m2m_links[key][comic_pk] = result
-
-    def _link_comic_m2m_fields(self) -> tuple[Mapping, int]:
-        """Get the complete m2m field data to create."""
-        total = 0
-        all_m2m_links = {}
-        comic_paths = frozenset(self.metadata.get(M2M_MDS, {}).keys())
-        if not comic_paths:
-            return all_m2m_links, total
-        comics = Comic.objects.filter(path__in=comic_paths).values_list("pk", "path")
-        for comic_pk, comic_path in comics:
-            md = self.metadata[M2M_MDS][comic_path]
-            link_data = (all_m2m_links, md, comic_pk)
-            self._link_prepare_special_m2ms(
-                link_data, FOLDERS_FIELD, Folder, self._get_link_folders_filter
-            )
-            for field_name, model in DICT_MODEL_FIELD_NAME_CLASS_MAP:
-                if model == Identifier:
-                    method = self._get_link_identifier_filter
-                else:
-                    method = self._get_link_dict_filter
-                self._link_prepare_special_m2ms(link_data, field_name, model, method)
-
-            total += self._link_named_m2ms(all_m2m_links, comic_pk, md)
-        self.metadata.pop(M2M_MDS)
-        return all_m2m_links, total
-
-    def _query_relation_adjustments(
-        self,
-        m2m_links,
-        ThroughModel,  # noqa:N803
-        through_field_id_name,
-        status,
-    ):
-        all_del_pks = set()
-        tms = []
-        for comic_pk, pks in m2m_links.items():
-            del_query = ~Q(**{through_field_id_name + "__in": pks})
-            del_pks = (
-                ThroughModel.objects.filter(comic_id=comic_pk)
-                .filter(del_query)
-                .values_list("pk", flat=True)
-            )
-            all_del_pks |= set(del_pks)
-            if status:
-                status.total = status.total or 0
-                status.total += len(del_pks)
-                self.status_controller.update(status)
-
-            extant_pks = set(
-                ThroughModel.objects.filter(comic_id=comic_pk).values_list(
-                    through_field_id_name, flat=True
-                )
-            )
-            missing_pks = set(pks) - extant_pks
-            if status:
-                status.total = status.total or 0
-                status.total += len(missing_pks)
-                self.status_controller.update(status)
-            for pk in missing_pks:
-                defaults = {"comic_id": comic_pk, through_field_id_name: pk}
-                tm = ThroughModel(**defaults)
-                tms.append(tm)
-        return tms, all_del_pks
-
-    def bulk_fix_comic_m2m_field(self, field_name, m2m_links, status):
-        """
-        Recreate an m2m field for a set of comics.
-
-        Since we can't bulk_update or bulk_create m2m fields use a trick.
-        bulk_create() on the through table:
-        https://stackoverflow.com/questions/6996176/how-to-create-an-object-for-a-django-model-with-a-many-to-many-field/10116452#10116452
-        https://docs.djangoproject.com/en/dev/ref/models/fields/#django.db.models.ManyToManyField.through
-        """
-        field = getattr(Comic, field_name)
-        ThroughModel = field.through  # noqa: N806
-
-        model = Comic._meta.get_field(field_name).related_model
-        if model is None:
-            reason = f"Bad model from {field_name}"
-            raise ValueError(reason)
-        link_name = model.__name__.lower()
-        through_field_id_name = f"{link_name}_id"
-
-        tms, all_del_pks = self._query_relation_adjustments(
-            m2m_links, ThroughModel, through_field_id_name, status
-        )
-        status.total = status.total or 0
-        status.total += len(tms) + len(all_del_pks)
-        self.status_controller.update(status)
-
-        update_fields = ("comic_id", through_field_id_name)
-        ThroughModel.objects.bulk_create(
-            tms,
-            update_conflicts=True,
-            update_fields=update_fields,
-            unique_fields=update_fields,
-        )
-        if created_count := len(tms):
-            self.log.info(f"Linked {created_count} new {field_name} to altered comics.")
-        status.complete = status.complete or 0
-        status.complete += created_count
-        self.status_controller.update(status)
-
-        if del_count := len(all_del_pks):
-            del_qs = ThroughModel.objects.filter(pk__in=all_del_pks)
-            del_qs.delete()
-            self.log.info(
-                f"Deleted {del_count} stale {field_name} relations for altered comics.",
-            )
-        status.complete = status.complete or 0
-        status.complete += del_count
-        self.status_controller.update(status)
-        return created_count, del_count
-
-    def bulk_query_and_link_comic_m2m_fields(self):
-        """Combine query and bulk link into a batch."""
-        status = Status(ImportStatusTypes.LINK_M2M_FIELDS, None, None)
-        self.status_controller.start(status)
-        all_m2m_links, link_total = self._link_comic_m2m_fields()
-        status.complete = 0
-        status.total = link_total
-        self.status_controller.update(status)
-        created_total = 0
-        del_total = 0
-        for field_name, m2m_links in all_m2m_links.items():
-            try:
-                created_count, del_count = self.bulk_fix_comic_m2m_field(
-                    field_name, m2m_links, status
-                )
-                created_total += created_count
-                del_total += del_count
-                status.complete += created_count + del_count
-                self.status_controller.update(status, notify=False)
-            except Exception:
-                self.log.exception(f"Error recreating m2m field: {field_name}")
-
-        if created_total:
-            self.log.info(f"Linked {created_total} comics to tags.")
-        if del_total:
-            self.log.info(f"Deleted {del_total} stale relations for altered comics.")
-        self.status_controller.finish(status)
-        return created_total + del_total

codex/librarian/importer/link_covers.py

@@ -1,73 +0,0 @@
-"""Link Covers."""
-
-from pathlib import Path
-
-from codex.librarian.importer.const import CLASS_CUSTOM_COVER_GROUP_MAP, LINK_COVER_PKS
-from codex.librarian.importer.failed_imports import FailedImportsImporter
-from codex.librarian.importer.status import ImportStatusTypes
-from codex.models import CustomCover, Folder
-from codex.status import Status
-
-
-class LinkCoversImporter(FailedImportsImporter):
-    """Link Covers methods."""
-
-    def _link_custom_cover_prepare(self, cover, model_map):
-        """Prepare one cover in the model map for bulk update."""
-        if cover.library and cover.library.covers_only:
-            model = CLASS_CUSTOM_COVER_GROUP_MAP.inverse.get(cover.group)
-            if not model:
-                self.log.warning(f"Custom Cover model not found for {cover.path}")
-                return
-            group_filter = {"sort_name": cover.sort_name}
-        else:
-            model = Folder
-            path = str(Path(cover.path).parent)
-            group_filter = {"path": path}
-        qs = model.objects.filter(**group_filter).exclude(custom_cover=cover)
-        if not qs.exists():
-            return
-
-        if model not in model_map:
-            model_map[model] = []
-
-        for obj in qs.iterator():
-            obj.custom_cover = cover
-            model_map[model].append(obj)
-
-    def _link_custom_cover_group(self, model, objs, status):
-        """Bulk link a group to it's custom covers."""
-        count = 0
-        if not objs:
-            return count
-        model.objects.bulk_update(objs, ["custom_cover"])
-        count += len(objs)
-        self.log.info(f"Linked {count} custom covers to {model.__name__}s")
-        status.complete += count
-        self.status_controller.update(status, notify=False)
-        return count
-
-    def link_custom_covers(self):
-        """Link Custom Covers to Groups."""
-        link_cover_pks = self.metadata.get(LINK_COVER_PKS, {})
-        num_link_cover_pks = len(link_cover_pks)
-        status = Status(ImportStatusTypes.COVERS_LINK, 0, num_link_cover_pks)
-        try:
-            if not num_link_cover_pks:
-                return 0
-            self.status_controller.start(status)
-            # Aggregate objs to update for each group model.
-            model_map = {}
-            covers = CustomCover.objects.filter(pk__in=link_cover_pks).only(
-                "library", "path"
-            )
-            for cover in covers:
-                self._link_custom_cover_prepare(cover, model_map)
-
-            # Bulk update each model type
-            total_count = 0
-            for model, objs in model_map.items():
-                total_count += self._link_custom_cover_group(model, objs, status)
-        finally:
-            self.status_controller.finish(status)
-        return total_count