codex 1.7.15__py3-none-any.whl → 1.8.0a2__py3-none-any.whl

codex/librarian/scribe/importer/moved/folders.py

@@ -0,0 +1,205 @@
+"""Bulk import and move comics and folders."""
+
+from pathlib import Path
+
+from bidict import bidict, frozenbidict
+from django.db.models.functions import Now
+
+from codex.librarian.scribe.importer.const import (
+    BULK_UPDATE_FOLDER_FIELDS,
+)
+from codex.librarian.scribe.importer.moved.covers import MovedCoversImporter
+from codex.librarian.scribe.importer.statii.create import ImporterCreateTagsStatus
+from codex.librarian.scribe.importer.statii.moved import ImporterMoveFoldersStatus
+from codex.librarian.status import Status
+from codex.models import Folder
+
+
+class MovedFoldersImporter(MovedCoversImporter):
+    """Methods for moving comics and folders."""
+
+    @staticmethod
+    def _folder_sort_key(element: Folder):
+        return len(Path(element.path).parts)
+
+    def _bulk_move_folders(
+        self,
+        src_folder_paths_with_existing_dest_parents,
+        dest_parent_folders_map,
+        dirs_moved: frozenbidict[str, str],
+        status: Status,
+    ):
+        """Bulk move folders."""
+        # Move collisions removed before this
+        folders_to_move = (
+            Folder.objects.filter(
+                library=self.library,
+                path__in=src_folder_paths_with_existing_dest_parents,
+            )
+            .only(*BULK_UPDATE_FOLDER_FIELDS)
+            .order_by("path")
+        )
+
+        new_paths = set()
+        update_folders = []
+        for folder in folders_to_move.iterator():
+            new_path = dirs_moved[folder.path]
+            new_paths.add(new_path)
+            folder.name = Path(new_path).name
+            folder.path = new_path
+            parent_path_str = str(Path(new_path).parent)
+            folder.parent_folder = dest_parent_folders_map.get(parent_path_str)
+            folder.presave()
+            folder.updated_at = Now()
+            update_folders.append(folder)
+
+        update_folders = sorted(update_folders, key=self._folder_sort_key)
+
+        Folder.objects.bulk_update(update_folders, BULK_UPDATE_FOLDER_FIELDS)
+        count = len(update_folders)
+        level = "INFO" if count else "DEBUG"
+        self.log.log(level, f"Moved {count} folders.")
+        status.increment_complete(count)
+        self.status_controller.update(status)
+        return count
+
+    def _bulk_move_folders_under_existing_parents(
+        self, dest_parent_folder_paths_map, dirs_moved: frozenbidict[str, str], status
+    ):
+        """Move folders under existing folders."""
+        count = 0
+        while True:
+            # Get existing parent folders
+            dest_parent_paths = tuple(dest_parent_folder_paths_map.keys())
+            extant_parent_folders = Folder.objects.filter(
+                library=self.library, path__in=dest_parent_paths
+            ).only("path")
+
+            # Create a list of folders than can be moved under existing folders.
+            dest_parent_folders_map = {}
+            src_folder_paths_with_existing_dest_parents = []
+            for extant_parent_folder in extant_parent_folders:
+                dest_parent_folders_map[extant_parent_folder.path] = (
+                    extant_parent_folder
+                )
+                if dest_folder_paths := dest_parent_folder_paths_map.pop(
+                    extant_parent_folder.path, None
+                ):
+                    for dest_folder_path in dest_folder_paths:
+                        src_path = dirs_moved.inverse[dest_folder_path]
+                        src_folder_paths_with_existing_dest_parents.append(src_path)
+
+            if not src_folder_paths_with_existing_dest_parents:
+                return count
+            src_folder_paths_with_existing_dest_parents = sorted(
+                src_folder_paths_with_existing_dest_parents
+            )
+            self.log.debug(
+                f"Moving folders under existing parents: {src_folder_paths_with_existing_dest_parents}"
+            )
+
+            count += self._bulk_move_folders(
+                src_folder_paths_with_existing_dest_parents,
+                dest_parent_folders_map,
+                dirs_moved,
+                status,
+            )
+
+    def _get_move_create_folders_one_layer(self, dest_parent_folder_paths_map):
+        """Find the next layer of folder paths to create."""
+        create_folder_paths_one_layer = set()
+        library_parts_len = len(Path(self.library.path).parts)
+        for parent_path_str in sorted(dest_parent_folder_paths_map.keys()):
+            parts = Path(parent_path_str).parts
+            # Get one layer of folders from existing layers.
+            possible_create_folder_path = ""
+            for index in range(library_parts_len, len(parts) + 1):
+                layer_parts = parts[:index]
+                possible_create_folder_path = str(Path(*layer_parts))
+                if (
+                    possible_create_folder_path in create_folder_paths_one_layer
+                    or not Folder.objects.filter(
+                        library=self.library, path=possible_create_folder_path
+                    ).exists()
+                ):
+                    break
+            else:
+                possible_create_folder_path = ""
+            if possible_create_folder_path:
+                create_folder_paths_one_layer.add(possible_create_folder_path)
+
+        return frozenset(create_folder_paths_one_layer)
+
+    def _remove_move_collisions(self, dirs_moved: bidict[str, str]):
+        """Remove moves that would collide with an existing Folder."""
+        dest_paths = set(dirs_moved.values())
+        collision_dest_paths = Folder.objects.filter(
+            library=self.library, path__in=dest_paths
+        ).values_list("path", flat=True)
+        if not collision_dest_paths:
+            return
+        collision_dest_paths = sorted(set(collision_dest_paths))
+        for collision_dest_path in collision_dest_paths:
+            dirs_moved.inverse.pop(collision_dest_path, None)
+        self.log.warning(
+            f"Not moving folders to destinations that would collide with existing database folders: {collision_dest_paths}"
+        )
+
+    def _bulk_move_folders_and_create_parents(self, status):
+        """Find folders that can be moved without creating parents."""
+        count = 0
+        dirs_moved = bidict(self.task.dirs_moved)
+
+        self._remove_move_collisions(dirs_moved)
+
+        dest_paths = sorted(set(dirs_moved.values()))
+        dest_parent_folder_paths_map = {}
+        for dest_path in dest_paths:
+            parent = str(Path(dest_path).parent)
+            if parent not in dest_parent_folder_paths_map:
+                dest_parent_folder_paths_map[parent] = set()
+            dest_parent_folder_paths_map[parent].add(dest_path)
+
+        create_status = ImporterCreateTagsStatus()
+        layer = 1
+        while True:
+            self._bulk_move_folders_under_existing_parents(
+                dest_parent_folder_paths_map, frozenbidict(dirs_moved), status
+            )
+
+            # All folders movable without creation have moved.
+            if not dest_parent_folder_paths_map:
+                break
+            self.log.debug(
+                f"Creating intermediate folder layer {layer} to move folders."
+            )
+            create_folder_paths_one_layer = self._get_move_create_folders_one_layer(
+                dest_parent_folder_paths_map
+            )
+
+            # Create one layer of folders
+            count += self.bulk_folders_create(
+                create_folder_paths_one_layer, create_status
+            )
+            layer += 1
+        return count
+
+    def bulk_folders_moved(self, *, mark_in_progress=False):
+        """Move folders in the database instead of recreating them."""
+        count = 0
+        num_dirs_moved = len(self.task.dirs_moved)
+        status = ImporterMoveFoldersStatus(None, num_dirs_moved)
+        try:
+            if not num_dirs_moved:
+                return count
+            if mark_in_progress:
+                self.library.start_update()
+            self.status_controller.start(status)
+
+            count += self._bulk_move_folders_and_create_parents(status)
+            self.task.dirs_moved = {}
+        finally:
+            self.status_controller.finish(status)
+            if mark_in_progress:
+                self.library.end_update()
+        return count

codex/librarian/scribe/importer/query/__init__.py

@@ -0,0 +1,44 @@
+"""Query missing foreign keys."""
+
+from codex.librarian.scribe.importer.const import (
+    CREATE_FKS,
+    DELETE_M2MS,
+    FTS_EXISTING_M2MS,
+    QUERY_MODELS,
+    UPDATE_COMICS,
+    UPDATE_FKS,
+)
+from codex.librarian.scribe.importer.query.links import QueryPruneLinks
+from codex.librarian.scribe.importer.statii.query import QUERY_STATII
+
+
+class QueryForeignKeysImporter(QueryPruneLinks):
+    """Methods for querying missing fks."""
+
+    def query(self):
+        """Get objects to create by querying existing objects for the proposed fks."""
+        if QUERY_MODELS not in self.metadata:
+            return
+        self.metadata[UPDATE_COMICS] = {}
+        self.metadata[CREATE_FKS] = {}
+        self.metadata[UPDATE_FKS] = {}
+        self.metadata[DELETE_M2MS] = {}
+        self.metadata[FTS_EXISTING_M2MS] = {}
+        self.log.debug(
+            f"Querying existing foreign keys for comics in {self.library.path}"
+        )
+        try:
+            if self.abort_event.is_set():
+                return
+            self.query_all_missing_models()
+            if self.abort_event.is_set():
+                return
+            self.query_update_comics()
+            if self.abort_event.is_set():
+                return
+            self.query_prune_comic_links()
+            if self.abort_event.is_set():
+                return
+            self.query_missing_custom_covers()
+        finally:
+            self.status_controller.finish_many(QUERY_STATII)

codex/librarian/scribe/importer/query/covers.py

@@ -0,0 +1,55 @@
+"""Query Missing Custom Covers."""
+
+from codex.librarian.scribe.importer.const import (
+    CREATE_COVERS,
+    UPDATE_COVERS,
+)
+from codex.librarian.scribe.importer.create import CreateForeignKeysImporter
+from codex.librarian.scribe.importer.statii.create import (
+    ImporterCreateCoversStatus,
+    ImporterUpdateCoversStatus,
+)
+from codex.librarian.scribe.importer.statii.query import (
+    ImporterQueryMissingCoversStatus,
+)
+from codex.models.paths import CustomCover
+
+
+class QueryCustomCoversImporter(CreateForeignKeysImporter):
+    """Query Missing Custom Covers."""
+
+    def query_missing_custom_covers(self):
+        """Identify update & create covers."""
+        cover_paths = self.task.covers_created | self.task.covers_modified
+        num_cover_paths = len(cover_paths)
+        status = ImporterQueryMissingCoversStatus(None, num_cover_paths)
+        if not num_cover_paths:
+            self.status_controller.finish(status)
+            return
+        self.log.debug(f"Querying {num_cover_paths} custom cover_paths")
+        self.status_controller.start(status)
+
+        update_covers_qs = CustomCover.objects.filter(
+            library=self.library, path__in=cover_paths
+        )
+        self.metadata[UPDATE_COVERS] = update_covers_qs
+        update_cover_paths = frozenset(update_covers_qs.values_list("path", flat=True))
+        update_count = len(update_cover_paths)
+        update_status = ImporterUpdateCoversStatus(0, update_count)
+        self.status_controller.update(update_status)
+
+        create_cover_paths = cover_paths - update_cover_paths
+        self.metadata[CREATE_COVERS] = create_cover_paths
+        create_count = len(create_cover_paths)
+        create_status = ImporterCreateCoversStatus(0, create_count)
+        self.status_controller.update(create_status)
+
+        self.task.covers_created = self.task.covers_modified = frozenset()
+
+        count = create_count + update_count
+        if count:
+            self.log.debug(
+                f"Discovered {update_count} custom covers to update and {create_count} to create."
+            )
+        self.status_controller.finish(status)
+        return

codex/librarian/scribe/importer/query/filters.py

@@ -0,0 +1,61 @@
+"""Query the missing foreign keys methods."""
+
+from django.db.models import Q
+
+from codex.librarian.scribe.importer.const import DictModelType
+from codex.librarian.scribe.importer.query.covers import QueryCustomCoversImporter
+from codex.models.base import BaseModel
+from codex.models.groups import BrowserGroupModel
+
+
+class QueryForeignKeysFilterImporter(QueryCustomCoversImporter):
+    """Query the missing foreign keys methods."""
+
+    @staticmethod
+    def _get_query_missing_simple_filter(
+        key_rels: tuple[str, ...],
+        key_values: tuple[str, ...],
+    ):
+        filter_args = {}
+        rel = key_rels[0]
+        values = frozenset(key[0] for key in key_values if key[0])
+        filter_args[f"{rel}__in"] = values
+        return Q(**filter_args)
+
+    @staticmethod
+    def _query_missing_complex_model_filter(
+        key_rels: tuple[str, ...],
+        key_values: tuple[tuple[str | None, ...], ...],
+    ):
+        """Add value filter to query filter map."""
+        query_filter = Q()
+        for keys in key_values:
+            filter_dict = {}
+            for rel, value in zip(key_rels, keys, strict=False):
+                # for credit
+                val = value[0] if isinstance(value, tuple) else value
+                if val is None:
+                    final_rel = rel + "__isnull"
+                    val = True
+                else:
+                    final_rel = rel
+                filter_dict[final_rel] = val
+            query_filter |= Q(**filter_dict)
+        return query_filter
+
+    def query_missing_model_filter(
+        self,
+        model: type[BaseModel],
+        key_rels: tuple[str, ...],
+        key_value_tuples: tuple,
+    ):
+        """Get filters for the model."""
+        if issubclass(model, DictModelType | BrowserGroupModel):
+            fk_filter = self._query_missing_complex_model_filter(
+                key_rels, key_value_tuples
+            )
+        else:
+            fk_filter = self._get_query_missing_simple_filter(
+                key_rels, key_value_tuples
+            )
+        return fk_filter

codex/librarian/scribe/importer/query/links.py

@@ -0,0 +1,29 @@
+"""Prune link actions."""
+
+from codex.librarian.scribe.importer.const import (
+    LINK_FKS,
+    LINK_M2MS,
+)
+from codex.librarian.scribe.importer.query.links_m2m import QueryPruneLinksM2M
+from codex.librarian.scribe.importer.statii.query import ImporterQueryTagLinksStatus
+
+
+class QueryPruneLinks(QueryPruneLinksM2M):
+    """Prune link actions."""
+
+    def query_prune_comic_links(self):
+        """Prune links that don't need updating."""
+        total_query_ops = self.sum_path_ops(LINK_FKS) + self.sum_path_ops(LINK_M2MS)
+        status = ImporterQueryTagLinksStatus(0, total_query_ops)
+        try:
+            if not total_query_ops:
+                return
+            self.status_controller.start(status)
+            if self.abort_event.is_set():
+                return
+            self.query_prune_comic_fk_links(status)
+            if self.abort_event.is_set():
+                return
+            self.query_prune_comic_m2m_links(status)
+        finally:
+            self.status_controller.finish(status)

codex/librarian/scribe/importer/query/links_fk.py

@@ -0,0 +1,99 @@
+"""Prune M2O links that don't need updating."""
+
+from codex.librarian.scribe.importer.const import (
+    COMIC_FK_FIELD_NAMES,
+    FIELD_NAME_KEYS_REL_MAP,
+    LINK_FKS,
+    PATH_FIELD_NAME,
+)
+from codex.librarian.scribe.importer.query.update_comics import QueryUpdateComics
+from codex.models.comic import Comic
+from codex.settings import LINK_FK_BATCH_SIZE
+
+_QUERY_LINK_FK_PRUNE_ONLY = (
+    PATH_FIELD_NAME,
+    *COMIC_FK_FIELD_NAMES,
+)
+
+
+class QueryPruneLinksFKs(QueryUpdateComics):
+    """Prune M2O links that don't need updating."""
+
+    def pop_links_to_fts(self, path, field_name):
+        """Pop a link to the FTS structure."""
+        link_key = self.metadata[LINK_FKS][path].pop(field_name)
+        self.add_links_to_fts(path, field_name, (link_key,))
+
+    def _query_prune_comic_fk_links_protagonist(
+        self, comic: Comic, path: str, field_name: str, key_values: tuple
+    ):
+        prot = key_values[0]
+        for obj in (comic.main_character, comic.main_team):
+            if obj and obj.name == prot:
+                self.metadata[LINK_FKS][path].pop(field_name)
+                break
+
+    @staticmethod
+    def _query_prune_comic_fk_links_key_equal(field_obj, key_rel, key_value):
+        parts = key_rel.split("__")
+        rel_obj = field_obj
+        key_val = None
+        while parts:
+            key_val = getattr(rel_obj, parts.pop(0), None)
+            rel_obj = key_val
+        return key_val == key_value
+
+    def _query_prune_comic_fk_links_field(self, comic, path, field_name):
+        link_dict = self.metadata[LINK_FKS].get(path)
+        key_values = link_dict[field_name]
+        if field_name == "protagonist":
+            self._query_prune_comic_fk_links_protagonist(
+                comic, path, field_name, key_values
+            )
+            return
+        key_rels = FIELD_NAME_KEYS_REL_MAP[field_name]
+        keys_equal = True
+        field_obj = getattr(comic, field_name)
+        for key_rel, key_value in zip(key_rels, key_values, strict=True):
+            keys_equal = self._query_prune_comic_fk_links_key_equal(
+                field_obj, key_rel, key_value
+            )
+            if not keys_equal:
+                break
+        if keys_equal:
+            self.metadata[LINK_FKS][path].pop(field_name)
+
+    def _query_prune_comic_fk_links_comic(self, comic, status):
+        path = comic.path
+        field_names = tuple(self.metadata[LINK_FKS].get(path).keys())
+        for field_name in field_names:
+            self._query_prune_comic_fk_links_field(comic, path, field_name)
+            status.increment_complete()
+            self.status_controller.update(status)
+        if not self.metadata[LINK_FKS][path]:
+            del self.metadata[LINK_FKS][path]
+
+    def _query_prune_comic_fk_links_batch(self, batch_paths: tuple[str, ...], status):
+        comics = (
+            Comic.objects.filter(library=self.library, path__in=batch_paths)
+            .select_related(*COMIC_FK_FIELD_NAMES)
+            .only(*_QUERY_LINK_FK_PRUNE_ONLY)
+        )
+        for comic in comics:
+            self._query_prune_comic_fk_links_comic(comic, status)
+
+    def query_prune_comic_fk_links(self, status):
+        """Prune comic fk links that already exist."""
+        status.subtitle = "Many to One"
+        self.status_controller.update(status)
+        paths = tuple(self.metadata[LINK_FKS].keys())
+        num_paths = len(paths)
+
+        start = 0
+        while start < num_paths:
+            if self.abort_event.is_set():
+                return
+            end = start + LINK_FK_BATCH_SIZE
+            batch_paths = paths[start:end]
+            self._query_prune_comic_fk_links_batch(batch_paths, status)
+            start += LINK_FK_BATCH_SIZE

codex/librarian/scribe/importer/query/links_m2m.py

@@ -0,0 +1,102 @@
+"""Prune M2M links that don't need updating."""
+
+from codex.librarian.scribe.importer.const import (
+    COMIC_M2M_FIELD_NAMES,
+    DELETE_M2MS,
+    FIELD_NAME_KEY_ATTRS_MAP,
+    LINK_M2MS,
+)
+from codex.librarian.scribe.importer.query.links_fk import QueryPruneLinksFKs
+from codex.models.base import BaseModel, NamedModel
+from codex.models.comic import Comic
+from codex.models.groups import BrowserGroupModel
+from codex.settings import LINK_M2M_BATCH_SIZE
+
+
+class QueryPruneLinksM2M(QueryPruneLinksFKs):
+    """Prune M2M links that don't need updating."""
+
+    @staticmethod
+    def _m2m_obj_to_key_tuple(key_attrs: tuple[str, ...], m2m_obj: BaseModel):
+        """Create a key value tuple from a db obj."""
+        value_tuple = []
+        for attr in key_attrs:
+            value = getattr(m2m_obj, attr)
+            if isinstance(value, NamedModel | BrowserGroupModel):
+                value = value.name
+            value_tuple.append(value)
+        return tuple(value_tuple)
+
+    def _query_prune_comic_m2m_links_field_obj(
+        self,
+        comic: Comic,
+        field_name: str,
+        m2m_obj: BaseModel,
+        kept_existing_values: set,
+        proposed_values: set,
+    ):
+        """Remove existing m2m links from the action list and add missing ones delete list."""
+        # transform objs into tuples
+        key_attrs = FIELD_NAME_KEY_ATTRS_MAP[field_name]
+        existing_key_value_tuple = self._m2m_obj_to_key_tuple(key_attrs, m2m_obj)
+        if existing_key_value_tuple in proposed_values:
+            # If already linked correctly, remove from action set
+            kept_existing_values.add(existing_key_value_tuple)
+            proposed_values.discard(existing_key_value_tuple)
+            deleted = False
+        else:
+            # Delete existing m2m links that aren't in the new comic
+            delete_m2m = (comic.pk, m2m_obj.pk)
+            self.metadata[DELETE_M2MS][field_name].add(delete_m2m)
+            deleted = True
+        return deleted
+
+    def _query_prune_comic_m2m_links_field(self, comic: Comic, field_name: str, status):
+        if field_name not in self.metadata[DELETE_M2MS]:
+            self.metadata[DELETE_M2MS][field_name] = set()
+
+        m2m_objs = getattr(comic, field_name).all()
+        deleted = False
+        kept_existing_values = set()
+        proposed_values = self.metadata[LINK_M2MS][comic.path][field_name]
+        for m2m_obj in m2m_objs:
+            deleted |= self._query_prune_comic_m2m_links_field_obj(
+                comic, field_name, m2m_obj, kept_existing_values, proposed_values
+            )
+            status.increment_complete()
+            self.status_controller.update(status)
+        if kept_existing_values and (deleted or proposed_values):
+            self.add_to_fts_existing(comic.pk, field_name, tuple(kept_existing_values))
+        if not self.metadata[LINK_M2MS][comic.path][field_name]:
+            del self.metadata[LINK_M2MS][comic.path][field_name]
+
+    def _query_prune_comic_m2m_links_comic(self, comic: Comic, status):
+        field_names = tuple(self.metadata[LINK_M2MS][comic.path].keys())
+        for field_name in field_names:
+            self._query_prune_comic_m2m_links_field(comic, field_name, status)
+        if not self.metadata[LINK_M2MS][comic.path]:
+            del self.metadata[LINK_M2MS][comic.path]
+
+    def _query_prune_comic_m2m_links_batch(self, paths: tuple[str], status):
+        comics = (
+            Comic.objects.filter(library=self.library, path__in=paths)
+            # prefetching deep links means a batch size of 190 or less
+            .prefetch_related(*COMIC_M2M_FIELD_NAMES)
+            .only(*COMIC_M2M_FIELD_NAMES)
+        )
+        for comic in comics.iterator(chunk_size=LINK_M2M_BATCH_SIZE):
+            self._query_prune_comic_m2m_links_comic(comic, status)
+
+    def query_prune_comic_m2m_links(self, status):
+        """Prune comic m2m links that already exists or should be deleted."""
+        status.subtitle = "Many to Many"
+        self.status_controller.update(status)
+        paths = tuple(self.metadata[LINK_M2MS].keys())
+        self._query_prune_comic_m2m_links_batch(paths, status)
+        field_names = tuple(self.metadata[DELETE_M2MS].keys())
+        for field_name in field_names:
+            rows = self.metadata[DELETE_M2MS][field_name]
+            if not rows:
+                del self.metadata[DELETE_M2MS][field_name]
+        status.increment_complete(len(field_names))
+        self.status_controller.update(status)