codex 1.7.14__py3-none-any.whl → 1.8.0__py3-none-any.whl

codex/librarian/scribe/importer/init.py

@@ -0,0 +1,338 @@
+"""Initiale Importer."""
+
+from dataclasses import asdict, dataclass
+from multiprocessing.queues import Queue
+from pathlib import Path
+from time import sleep, time
+from typing import Any
+
+from django.utils.timezone import now
+
+from codex.librarian.scribe.importer.statii.create import (
+    ImporterCreateComicsStatus,
+    ImporterCreateCoversStatus,
+    ImporterCreateTagsStatus,
+    ImporterUpdateComicsStatus,
+    ImporterUpdateCoversStatus,
+    ImporterUpdateTagsStatus,
+)
+from codex.librarian.scribe.importer.statii.delete import (
+    ImporterRemoveComicsStatus,
+    ImporterRemoveCoversStatus,
+    ImporterRemoveFoldersStatus,
+)
+from codex.librarian.scribe.importer.statii.link import (
+    ImporterLinkCoversStatus,
+    ImporterLinkTagsStatus,
+)
+from codex.librarian.scribe.importer.statii.moved import (
+    ImporterMoveComicsStatus,
+    ImporterMoveCoversStatus,
+    ImporterMoveFoldersStatus,
+)
+from codex.librarian.scribe.importer.statii.query import (
+    ImporterQueryComicUpdatesStatus,
+    ImporterQueryMissingCoversStatus,
+    ImporterQueryMissingTagsStatus,
+    ImporterQueryTagLinksStatus,
+)
+from codex.librarian.scribe.importer.statii.read import (
+    ImporterAggregateStatus,
+    ImporterReadComicsStatus,
+)
+from codex.librarian.scribe.importer.statii.search import (
+    ImporterFTSCreateStatus,
+    ImporterFTSUpdateStatus,
+)
+from codex.librarian.scribe.importer.tasks import ImportTask
+from codex.librarian.scribe.search.status import SearchIndexCleanStatus
+from codex.librarian.scribe.status import UpdateGroupTimestampsStatus
+from codex.librarian.worker import WorkerStatusMixin
+from codex.models import Library
+from codex.settings import LOGLEVEL
+
+_WRITE_WAIT_EXPIRY = 60
+
+
+@dataclass
+class Counts:
+    """Total counts of operations."""
+
+    comic: int = 0
+    tags: int = 0
+    link: int = 0
+    link_covers: int = 0
+    folders: int = 0
+    comics_deleted: int = 0
+    folders_deleted: int = 0
+    tags_deleted: int = 0
+    covers: int = 0
+    covers_deleted: int = 0
+    comics_moved: int = 0
+    folders_moved: int = 0
+    covers_moved: int = 0
+    failed_imports: int = 0
+
+    def _any(self, exclude_prefixes: tuple[str, ...]):
+        return any(
+            value
+            for key, value in asdict(self).items()
+            if not key.startswith(exclude_prefixes)
+        )
+
+    def changed(self):
+        """Anything changed at all."""
+        return self._any(("failed",))
+
+    def search_changed(self):
+        """Is the search index be out of date."""
+        return self._any(
+            (
+                "cover",
+                "failed",
+            )
+        )
+
+
+class InitImporter(WorkerStatusMixin):
+    """Initial Importer."""
+
+    def __init__(
+        self, task: ImportTask, logger_, librarian_queue: Queue, db_write_lock, event
+    ):
+        """Initialize the import."""
+        self.init_worker(logger_, librarian_queue, db_write_lock)
+        self.task: ImportTask = task
+        self.metadata: dict[str, Any] = {}
+        self.counts = Counts()
+        self.library = Library.objects.only("path").get(pk=self.task.library_id)
+        self.abort_event = event
+        self.start_time = now()
+        self._is_log_debug_task = (
+            self.log.level(LOGLEVEL).no <= self.log.level("DEBUG").no
+        )
+
+    def _wait_for_filesystem_ops_to_finish(self) -> bool:
+        """Watchdog sends events before filesystem events finish, so wait for them."""
+        started_checking = time()
+
+        # Don't wait for deletes to complete.
+        # Do wait for move, modified, create files before import.
+        all_modified_paths = (
+            frozenset(self.task.dirs_moved.values())
+            | frozenset(self.task.files_moved.values())
+            | self.task.dirs_modified
+            | self.task.files_modified
+            | self.task.files_created
+        )
+
+        old_total_size = -1
+        total_size = 0
+        wait_time = 2
+        while old_total_size != total_size:
+            if old_total_size > 0:
+                # second time around or more
+                sleep(wait_time)
+                wait_time = wait_time**2
+                reason = (
+                    f"Waiting for files to copy before import: "
+                    f"{old_total_size} != {total_size}"
+                )
+                self.log.debug(reason)
+            if time() - started_checking > _WRITE_WAIT_EXPIRY:
+                return True
+
+            old_total_size = total_size
+            total_size = 0
+            for path_str in all_modified_paths:
+                path = Path(path_str)
+                if path.exists():
+                    total_size += Path(path).stat().st_size
+        return False
+
+    #######
+    # LOG #
+    #######
+    def _log_task_construct_dirs_log(self):
+        """Construct dirs log line."""
+        dirs_log = []
+        if self.task.dirs_moved:
+            dirs_log += [f"{len(self.task.dirs_moved)} moved"]
+        if self.task.dirs_modified:
+            dirs_log += [f"{len(self.task.dirs_modified)} modified"]
+        if self.task.dirs_deleted:
+            dirs_log += [f"{len(self.task.dirs_deleted)} deleted"]
+        return dirs_log
+
+    def _log_task_construct_comics_log(self):
+        """Construct comcis log line."""
+        comics_log = []
+        if self.task.files_moved:
+            comics_log += [f"{len(self.task.files_moved)} moved"]
+        if self.task.files_modified:
+            comics_log += [f"{len(self.task.files_modified)} modified"]
+        if self.task.files_created:
+            comics_log += [f"{len(self.task.files_created)} created"]
+        if self.task.files_deleted:
+            comics_log += [f"{len(self.task.files_deleted)} deleted"]
+        return comics_log
+
+    def _log_task(self):
+        """Log the watchdog event self.task."""
+        if not self._is_log_debug_task:
+            return
+
+        self.log.debug(f"Updating library {self.library.path}...")
+        comics_log = self._log_task_construct_comics_log()
+        if comics_log:
+            log = "Comics: "
+            log += ", ".join(comics_log)
+            self.log.debug("  " + log)
+
+        dirs_log = self._log_task_construct_dirs_log()
+        if dirs_log:
+            log = "Folders: "
+            log += ", ".join(dirs_log)
+            self.log.debug("  " + log)
+
+    ########
+    # INIT #
+    ########
+    def _init_librarian_status_moved(self, status_list):
+        """Initialize moved statuses."""
+        search_index_updates = 0
+        if self.task.dirs_moved:
+            status_list += [ImporterMoveFoldersStatus(None, len(self.task.dirs_moved))]
+        if self.task.files_moved:
+            status_list += [ImporterMoveComicsStatus(None, len(self.task.files_moved))]
+            search_index_updates += len(self.task.files_moved)
+        if self.task.covers_moved:
+            status_list += [ImporterMoveCoversStatus(None, len(self.task.covers_moved))]
+        return search_index_updates
+
+    def _init_if_modified_or_created(self, path, status_list):
+        """Initialize librarian statuses for modified or created ops."""
+        total_paths = len(self.task.files_modified) + len(self.task.files_created)
+        status_list += [
+            ImporterReadComicsStatus(0, total_paths, subtitle=path),
+            ImporterAggregateStatus(0, total_paths, subtitle=path),
+            ImporterQueryMissingTagsStatus(subtitle=path),
+            ImporterQueryComicUpdatesStatus(subtitle=path),
+            ImporterQueryTagLinksStatus(subtitle=path),
+            ImporterQueryMissingCoversStatus(subtitle=path),
+            ImporterCreateTagsStatus(subtitle=path),
+            ImporterUpdateTagsStatus(subtitle=path),
+        ]
+        if self.task.covers_modified:
+            status_list += [
+                ImporterUpdateCoversStatus(
+                    None,
+                    len(self.task.covers_modified),
+                    subtitle=path,
+                )
+            ]
+        if self.task.covers_created:
+            status_list += [
+                ImporterCreateCoversStatus(
+                    None,
+                    len(self.task.covers_created),
+                    subtitle=path,
+                )
+            ]
+        if self.task.files_created or self.task.covers_created:
+            status_list += [
+                ImporterCreateComicsStatus(
+                    None,
+                    len(self.task.files_created),
+                    subtitle=path,
+                )
+            ]
+        if self.task.files_modified:
+            status_list += [
+                ImporterUpdateComicsStatus(
+                    None,
+                    len(self.task.files_modified),
+                    subtitle=path,
+                )
+            ]
+        if self.task.files_modified or self.task.files_created:
+            status_list += [ImporterLinkTagsStatus(subtitle=path)]
+
+        num_covers_linked = (
+            len(self.task.covers_moved)
+            + len(self.task.covers_modified)
+            + len(self.task.covers_created)
+        )
+        if num_covers_linked:
+            status_list += [
+                ImporterLinkCoversStatus(
+                    None,
+                    num_covers_linked,
+                    subtitle=path,
+                )
+            ]
+
+        modified = len(self.task.files_moved) + len(self.task.files_modified)
+        created = len(self.task.files_created)
+
+        return modified, created
+
+    def _init_librarian_status_deleted(self, status_list):
+        """Init deleted statuses."""
+        search_index_updates = 0
+        if self.task.files_deleted:
+            status_list += [
+                ImporterRemoveComicsStatus(None, len(self.task.files_deleted))
+            ]
+            search_index_updates += len(self.task.files_deleted)
+        if self.task.dirs_deleted:
+            status_list += [
+                ImporterRemoveFoldersStatus(None, len(self.task.dirs_deleted))
+            ]
+        if self.task.covers_deleted:
+            status_list += [
+                ImporterRemoveCoversStatus(None, len(self.task.covers_deleted))
+            ]
+        return search_index_updates
+
+    @staticmethod
+    def _init_librarian_status_search_index(
+        comic_updates, comic_creates, comic_deletes, status_list
+    ):
+        """Init search index statuses."""
+        status_list += [
+            SearchIndexCleanStatus(total=comic_deletes),
+            ImporterFTSUpdateStatus(total=comic_updates),
+            ImporterFTSCreateStatus(total=comic_creates),
+        ]
+
+    def _init_librarian_status(self, path):
+        """Update the librarian status self.tasks."""
+        status_list = []
+        moved = self._init_librarian_status_moved(status_list)
+        modified, created = self._init_if_modified_or_created(path, status_list)
+        deleted = self._init_librarian_status_deleted(status_list)
+        status_list += [UpdateGroupTimestampsStatus()]
+        self._init_librarian_status_search_index(
+            modified + moved,
+            created,
+            deleted,
+            status_list,
+        )
+        self.status_controller.start_many(status_list)
+
+    def init_apply(self):
+        """Initialize the library and status flags."""
+        self.start_time = now()
+        self.library.start_update()
+        too_long = self._wait_for_filesystem_ops_to_finish()
+        if too_long:
+            reason = (
+                "Import apply waited for the filesystem to stop changing too long. "
+                "Try polling again once files have finished copying"
+                f" in library: {self.library.path}"
+            )
+            self.log.warning(reason)
+            return
+        self._log_task()
+        self._init_librarian_status(self.library.path)

codex/librarian/scribe/importer/link/__init__.py

@@ -0,0 +1,15 @@
+"""Bulk update m2m fields."""
+
+from codex.librarian.scribe.importer.link.many_to_many import LinkManyToManyImporter
+
+
+class LinkComicsImporter(LinkManyToManyImporter):
+    """Link comics methods."""
+
+    def link(self):
+        """Link tags and covers."""
+        self.counts.link += self.link_comic_m2m_fields()
+        if self.abort_event.is_set():
+            return
+        if count := self.link_custom_covers():
+            self.counts.link_covers += count

codex/librarian/scribe/importer/link/const.py

@@ -0,0 +1,31 @@
+"""Link constants."""
+
+from types import MappingProxyType
+
+from codex.librarian.scribe.importer.const import (
+    CREDITS_FIELD_NAME,
+    IDENTIFIERS_FIELD_NAME,
+    STORY_ARC_NUMBERS_FIELD_NAME,
+)
+from codex.models.base import BaseModel
+from codex.models.groups import Imprint, Series, Volume
+
+COMPLEX_MODEL_FIELD_NAMES = (
+    CREDITS_FIELD_NAME,
+    STORY_ARC_NUMBERS_FIELD_NAME,
+    IDENTIFIERS_FIELD_NAME,
+)
+DEFAULT_KEY_RELS = ("name",)
+GROUP_KEY_RELS: MappingProxyType[type[BaseModel], tuple[str, ...]] = MappingProxyType(
+    {
+        Imprint: ("publisher__name", "name"),
+        Series: ("publisher__name", "imprint__name", "name"),
+        Volume: (
+            "publisher__name",
+            "imprint__name",
+            "series__name",
+            "name",
+            "number_to",
+        ),
+    }
+)

codex/librarian/scribe/importer/link/covers.py

@@ -0,0 +1,70 @@
+"""Link Covers."""
+
+from pathlib import Path
+
+from codex.librarian.scribe.importer.const import (
+    CLASS_CUSTOM_COVER_GROUP_MAP,
+    LINK_COVER_PKS,
+)
+from codex.librarian.scribe.importer.failed.failed import FailedImportsImporter
+from codex.librarian.scribe.importer.statii.link import ImporterLinkCoversStatus
+from codex.models import CustomCover, Folder
+
+
+class LinkCoversImporter(FailedImportsImporter):
+    """Link Covers methods."""
+
+    def _link_custom_cover_prepare(self, cover, model_map):
+        """Prepare one cover in the model map for bulk update."""
+        if cover.library and cover.library.covers_only:
+            model = CLASS_CUSTOM_COVER_GROUP_MAP.inverse.get(cover.group)
+            if not model:
+                self.log.warning(f"Custom Cover model not found for {cover.path}")
+                return
+            group_filter = {"sort_name": cover.sort_name}
+        else:
+            model = Folder
+            path = str(Path(cover.path).parent)
+            group_filter = {"path": path}
+        qs = model.objects.filter(**group_filter).exclude(custom_cover=cover)
+        if not qs.exists():
+            return
+
+        if model not in model_map:
+            model_map[model] = []
+
+        for obj in qs.iterator():
+            obj.custom_cover = cover
+            model_map[model].append(obj)
+
+    def _link_custom_cover_group(self, model, objs, status):
+        """Bulk link a group to it's custom covers."""
+        if not objs:
+            return
+        model.objects.bulk_update(objs, ["custom_cover"])
+        status.complete += len(objs)
+        self.status_controller.update(status)
+
+    def link_custom_covers(self):
+        """Link Custom Covers to Groups."""
+        link_cover_pks = self.metadata.get(LINK_COVER_PKS, {})
+        num_link_cover_pks = len(link_cover_pks)
+        status = ImporterLinkCoversStatus(0, num_link_cover_pks)
+        try:
+            if not num_link_cover_pks:
+                return 0
+            self.status_controller.start(status)
+            # Aggregate objs to update for each group model.
+            model_map = {}
+            covers = CustomCover.objects.filter(pk__in=link_cover_pks).only(
+                "library", "path"
+            )
+            for cover in covers:
+                self._link_custom_cover_prepare(cover, model_map)
+
+            # Bulk update each model type
+            for model, objs in model_map.items():
+                self._link_custom_cover_group(model, objs, status)
+        finally:
+            self.status_controller.finish(status)
+        return status.complete

codex/librarian/scribe/importer/link/delete.py

@@ -0,0 +1,106 @@
+"""Delete stale m2ms."""
+
+from typing import TYPE_CHECKING
+
+from django.db.models import ManyToManyField, Q
+
+from codex.librarian.scribe.importer.const import (
+    DELETE_M2MS,
+    FTS_EXISTING_M2MS,
+    FTS_UPDATE,
+)
+from codex.librarian.scribe.importer.link.prepare import LinkComicsImporterPrepare
+from codex.models import Comic
+from codex.models.base import BaseModel
+from codex.settings import FILTER_BATCH_SIZE
+
+if TYPE_CHECKING:
+    from django.db.models.fields.related import ManyToManyRel
+
+
+class LinkImporterDelete(LinkComicsImporterPrepare):
+    """Delete stale m2ms."""
+
+    @staticmethod
+    def get_through_model(field: ManyToManyField) -> type[BaseModel]:
+        """Get the through model for a m2m field."""
+        remote_field: ManyToManyRel = field.remote_field  # pyright: ignore[reportAssignmentType]
+        return remote_field.through  #  pyright: ignore[reportReturnType]
+
+    def _delete_m2m_field_batch(
+        self,
+        column_name: str,
+        through_model: type[BaseModel],
+        batch_rows: tuple,
+        comic_ids: set[int],
+    ):
+        del_filter = Q()
+        for row in batch_rows:
+            comic_id, model_id = row
+            del_filter_dict = {"comic_id": comic_id, column_name: model_id}
+            del_filter |= Q(**del_filter_dict)
+            comic_ids.add(comic_id)
+
+        qs = through_model.objects.filter(del_filter)
+        count, _ = qs.delete()
+        return count
+
+    def _delete_m2m_fts_entries(self, field_name: str, comic_ids: set[int]):
+        fts_field_name = (
+            "story_arcs" if field_name == "story_arc_numbers" else field_name
+        )
+        for comic_id in comic_ids:
+            if not self.metadata[FTS_UPDATE].get(comic_id, {}).get(
+                fts_field_name
+            ) and not self.metadata[FTS_EXISTING_M2MS].get(comic_id, {}).get(
+                fts_field_name
+            ):
+                self.add_links_to_fts(comic_id, fts_field_name, ())
+
+    def delete_m2m_field(self, field_name: str, delete_m2ms: dict, status):
+        """Delete one comic field's m2m relations."""
+        total_field_count = 0
+        rows = tuple(delete_m2ms.pop(field_name, ()))
+        num_rows = len(rows)
+        if not num_rows:
+            return total_field_count
+        status.subtitle = f"Delete stale {field_name} links"
+        self.status_controller.update(status)
+        field: ManyToManyField = Comic._meta.get_field(field_name)  # pyright:ignore[reportAssignmentType], # ty: ignore[invalid-assignment]│
+        related_model = field.related_model
+        table_name = related_model._meta.db_table
+        column_name = table_name.removeprefix("codex_") + "_id"
+        through_model = self.get_through_model(field)
+        comic_ids = set()
+
+        start = 0
+        while start < num_rows:
+            end = start + FILTER_BATCH_SIZE
+            batch_rows = rows[start:end]
+            count = self._delete_m2m_field_batch(
+                column_name, through_model, batch_rows, comic_ids
+            )
+            status.complete += count
+            total_field_count += count
+            if count:
+                self.log.info(
+                    f"Deleted {total_field_count}/{num_rows} stale {field_name} relations for altered comics.",
+                )
+
+            self.status_controller.update(status)
+            start += FILTER_BATCH_SIZE
+
+        self._delete_m2m_fts_entries(field_name, comic_ids)
+
+        return total_field_count
+
+    def delete_m2ms(self, status):
+        """Delete old missing m2ms."""
+        delete_m2ms = self.metadata.pop(DELETE_M2MS, {})
+
+        del_total = 0
+        for field_name in sorted(delete_m2ms.keys()):
+            del_total += self.delete_m2m_field(field_name, delete_m2ms, status)
+        if del_total:
+            self.log.info(f"Deleted {del_total} stale relations for altered comics.")
+        return del_total

codex/librarian/scribe/importer/link/many_to_many.py

@@ -0,0 +1,88 @@
+"""Link Comics M2M fields."""
+
+from typing import TYPE_CHECKING
+
+from codex.librarian.scribe.importer.const import (
+    DELETE_M2MS,
+    LINK_M2MS,
+)
+from codex.librarian.scribe.importer.link.sum import LinkSumImporter
+from codex.librarian.scribe.importer.statii.link import ImporterLinkTagsStatus
+from codex.librarian.status import Status
+from codex.models import Comic
+
+if TYPE_CHECKING:
+    from django.db.models import ManyToManyField
+
+
+class LinkManyToManyImporter(LinkSumImporter):
+    """Link Comics M2M fields."""
+
+    def link_comic_m2m_field(self, field_name, m2m_links, status: Status):
+        """
+        Recreate an m2m field for a set of comics.
+
+        Since we can't bulk_update or bulk_create m2m fields use a trick.
+        bulk_create() on the through table:
+        https://stackoverflow.com/questions/6996176/how-to-create-an-object-for-a-django-model-with-a-many-to-many-field/10116452#10116452
+        https://docs.djangoproject.com/en/dev/ref/models/fields/#django.db.models.ManyToManyField.through
+        """
+        status.subtitle = field_name
+        self.status_controller.update(status)
+        field: ManyToManyField = Comic._meta.get_field(field_name)  # pyright: ignore[reportAssignmentType], # ty: ignore[invalid-assignment]│
+        through_model = self.get_through_model(field)
+        through_field_id_name = field.related_model.__name__.lower() + "_id"
+
+        tms = []
+        for comic_id, model_ids in m2m_links.items():
+            for model_id in model_ids:
+                args = {"comic_id": comic_id, through_field_id_name: model_id}
+                tm = through_model(**args)
+                tms.append(tm)
+
+        count = len(tms)
+        if count:
+            update_fields = ("comic_id", through_field_id_name)
+            through_model.objects.bulk_create(
+                tms,
+                update_conflicts=True,
+                update_fields=update_fields,
+                unique_fields=update_fields,
+            )
+            status.increment_complete(count)
+            self.status_controller.update(status)
+        return count
+
+    def link_comic_m2m_fields(self):
+        """Combine query and bulk link into a batch."""
+        link_total = self.sum_ops(DELETE_M2MS) + self.sum_path_ops(LINK_M2MS)
+        status = ImporterLinkTagsStatus(0, link_total)
+        try:
+            if not link_total:
+                self.status_controller.finish(status)
+                return link_total
+
+            self.status_controller.start(status)
+            del_total = self.delete_m2ms(status)
+
+            # get ids for through model writing.
+            all_m2m_links = self.link_prepare_m2m_links(status)
+
+            num_links = sum(
+                len(m2m_links.values()) for m2m_links in all_m2m_links.values()
+            )
+            status.total = num_links
+
+            created_total = 0
+            for field_name, m2m_links in all_m2m_links.items():
+                if self.abort_event.is_set():
+                    return created_total + del_total
+                try:
+                    created_total += self.link_comic_m2m_field(
+                        field_name, m2m_links, status
+                    )
+                except Exception:
+                    self.log.exception(f"Error recreating m2m field: {field_name}")
+        finally:
+            self.status_controller.finish(status)
+        return created_total + del_total