codex 1.7.14__py3-none-any.whl → 1.8.0__py3-none-any.whl

codex/librarian/scribe/importer/query/covers.py

@@ -0,0 +1,55 @@
+"""Query Missing Custom Covers."""
+
+from codex.librarian.scribe.importer.const import (
+    CREATE_COVERS,
+    UPDATE_COVERS,
+)
+from codex.librarian.scribe.importer.create import CreateForeignKeysImporter
+from codex.librarian.scribe.importer.statii.create import (
+    ImporterCreateCoversStatus,
+    ImporterUpdateCoversStatus,
+)
+from codex.librarian.scribe.importer.statii.query import (
+    ImporterQueryMissingCoversStatus,
+)
+from codex.models.paths import CustomCover
+
+
+class QueryCustomCoversImporter(CreateForeignKeysImporter):
+    """Query Missing Custom Covers."""
+
+    def query_missing_custom_covers(self):
+        """Identify update & create covers."""
+        cover_paths = self.task.covers_created | self.task.covers_modified
+        num_cover_paths = len(cover_paths)
+        status = ImporterQueryMissingCoversStatus(None, num_cover_paths)
+        if not num_cover_paths:
+            self.status_controller.finish(status)
+            return
+        self.log.debug(f"Querying {num_cover_paths} custom cover_paths")
+        self.status_controller.start(status)
+
+        update_covers_qs = CustomCover.objects.filter(
+            library=self.library, path__in=cover_paths
+        )
+        self.metadata[UPDATE_COVERS] = update_covers_qs
+        update_cover_paths = frozenset(update_covers_qs.values_list("path", flat=True))
+        update_count = len(update_cover_paths)
+        update_status = ImporterUpdateCoversStatus(0, update_count)
+        self.status_controller.update(update_status)
+
+        create_cover_paths = cover_paths - update_cover_paths
+        self.metadata[CREATE_COVERS] = create_cover_paths
+        create_count = len(create_cover_paths)
+        create_status = ImporterCreateCoversStatus(0, create_count)
+        self.status_controller.update(create_status)
+
+        self.task.covers_created = self.task.covers_modified = frozenset()
+
+        count = create_count + update_count
+        if count:
+            self.log.debug(
+                f"Discovered {update_count} custom covers to update and {create_count} to create."
+            )
+        self.status_controller.finish(status)
+        return

codex/librarian/scribe/importer/query/filters.py

@@ -0,0 +1,61 @@
+"""Query the missing foreign keys methods."""
+
+from django.db.models import Q
+
+from codex.librarian.scribe.importer.const import DictModelType
+from codex.librarian.scribe.importer.query.covers import QueryCustomCoversImporter
+from codex.models.base import BaseModel
+from codex.models.groups import BrowserGroupModel
+
+
+class QueryForeignKeysFilterImporter(QueryCustomCoversImporter):
+    """Query the missing foreign keys methods."""
+
+    @staticmethod
+    def _get_query_missing_simple_filter(
+        key_rels: tuple[str, ...],
+        key_values: tuple[str, ...],
+    ):
+        filter_args = {}
+        rel = key_rels[0]
+        values = frozenset(key[0] for key in key_values if key[0])
+        filter_args[f"{rel}__in"] = values
+        return Q(**filter_args)
+
+    @staticmethod
+    def _query_missing_complex_model_filter(
+        key_rels: tuple[str, ...],
+        key_values: tuple[tuple[str | None, ...], ...],
+    ):
+        """Add value filter to query filter map."""
+        query_filter = Q()
+        for keys in key_values:
+            filter_dict = {}
+            for rel, value in zip(key_rels, keys, strict=False):
+                # for credit
+                val = value[0] if isinstance(value, tuple) else value
+                if val is None:
+                    final_rel = rel + "__isnull"
+                    val = True
+                else:
+                    final_rel = rel
+                filter_dict[final_rel] = val
+            query_filter |= Q(**filter_dict)
+        return query_filter
+
+    def query_missing_model_filter(
+        self,
+        model: type[BaseModel],
+        key_rels: tuple[str, ...],
+        key_value_tuples: tuple,
+    ):
+        """Get filters for the model."""
+        if issubclass(model, DictModelType | BrowserGroupModel):
+            fk_filter = self._query_missing_complex_model_filter(
+                key_rels, key_value_tuples
+            )
+        else:
+            fk_filter = self._get_query_missing_simple_filter(
+                key_rels, key_value_tuples
+            )
+        return fk_filter

codex/librarian/scribe/importer/query/foreign_keys.py

@@ -0,0 +1,203 @@
+"""Query the missing foreign keys methods."""
+
+from codex.librarian.scribe.importer.const import (
+    CREATE_FKS,
+    FK_KEYS,
+    MODEL_REL_MAP,
+    MODEL_SELECT_RELATED,
+    QUERY_MODELS,
+    TOTAL,
+    UPDATE_FKS,
+    get_key_index,
+)
+from codex.librarian.scribe.importer.query.update_fks import QueryIsUpdateImporter
+from codex.librarian.scribe.importer.statii.create import (
+    ImporterCreateTagsStatus,
+    ImporterUpdateTagsStatus,
+)
+from codex.librarian.scribe.importer.statii.query import ImporterQueryMissingTagsStatus
+from codex.librarian.status import Status
+from codex.models.base import BaseModel
+from codex.models.named import Universe
+from codex.settings import FILTER_BATCH_SIZE
+from codex.util import flatten
+
+
+class QueryForeignKeysQueryImporter(QueryIsUpdateImporter):
+    """Query the missing foreign keys methods."""
+
+    def query_existing_mds(self, model: type[BaseModel], batch_proposed_key_tuples):
+        """Query existing metadata tables."""
+        key_rels: tuple[str, ...] = MODEL_REL_MAP[model][0]  # pyright:ignore[reportAssignmentType], # ty: ignore[invalid-assignment]
+        fk_filter = self.query_missing_model_filter(
+            model,
+            key_rels,
+            batch_proposed_key_tuples,
+        )
+        rels = MODEL_REL_MAP[model]
+        select_related = MODEL_SELECT_RELATED.get(model, ())
+        fields = tuple(filter(bool, flatten(rels)))
+        qs = model.objects
+        qs = qs.select_related(*select_related)
+        qs = qs.filter(fk_filter).distinct().values_list(*fields)
+        extra_index = get_key_index(model)
+        has_id = bool(rels[1])
+        existing_mds = {}
+        for existing_values in qs:
+            key = existing_values[:extra_index]
+            value = existing_values[extra_index:]
+            if has_id:
+                identifier = value[:3] if any(value[:3]) else None
+                value = (identifier, *value[3:])
+            existing_mds[key] = value
+        return existing_mds
+
+    def _query_missing_models_batch(  # noqa: PLR0913
+        self,
+        model: type[BaseModel],
+        start: int,
+        all_proposed_key_values: tuple,
+        proposed_values_map: dict[tuple, set[tuple]],
+        create_values: set[tuple],
+        update_values: set[tuple],
+        fts_values: dict[tuple, tuple],
+        status: Status,
+    ):
+        # Do this in batches so as not to exceed the 1k sqlite query depth limit
+        end = start + FILTER_BATCH_SIZE
+        batch_proposed_key_tuples = all_proposed_key_values[start:end]
+        num_in_batch = len(batch_proposed_key_tuples)
+
+        existing_values_map = self.query_existing_mds(model, batch_proposed_key_tuples)
+
+        for key_values in batch_proposed_key_tuples:
+            proposed_extra_values_set = proposed_values_map.pop(key_values)
+            exists = key_values in existing_values_map
+            existing_extra_values = existing_values_map.pop(key_values, None)
+
+            do_update, best_extra_values = self.query_model_best_extra_values(
+                model, existing_extra_values, proposed_extra_values_set
+            )
+            best_values = (*key_values, *best_extra_values)
+            if exists:
+                if do_update:
+                    update_values.add(best_values)
+            else:
+                create_values.add(best_values)
+            if model is Universe:
+                fts_values[key_values] = best_extra_values
+
+        status.increment_complete(num_in_batch)
+        self.status_controller.update(status)
+
+    def _finish_query_missing(
+        self,
+        model: type[BaseModel],
+        values: set | frozenset | dict,
+        key: str,
+        title: str,
+    ):
+        if values:
+            fks = self.metadata[key]
+            if isinstance(values, dict):
+                if model not in fks:
+                    fks[model] = {}
+                fks[model].update(values)
+            else:
+                if model not in fks:
+                    fks[model] = set()
+                fks[model] |= values
+            level = "INFO"
+        else:
+            level = "DEBUG"
+        num = len(values)
+        match key:
+            case FK_KEYS.CREATE_FKS:
+                verb = "create"
+            case FK_KEYS.UPDATE_FKS:
+                verb = "update"
+            case _:
+                verb = ""
+        if verb:
+            self.log.log(level, f"Prepared {num} {title} for {verb}.")
+
+    def query_missing_models(
+        self,
+        model: type[BaseModel],
+        proposed_values_map: dict[tuple, set[tuple]],
+        status: Status,
+    ):
+        """Find missing foreign key models."""
+        if not proposed_values_map:
+            return 0
+        num_all_proposed_values = len(proposed_values_map)
+        proposed_key_values = tuple(proposed_values_map.keys())
+        start = 0
+
+        vnp = model._meta.verbose_name_plural
+        title = vnp.title() if vnp else ""
+        status.subtitle = title
+
+        create_values = set()
+        update_values = set()
+        fts_values = {}
+
+        while start < num_all_proposed_values:
+            self._query_missing_models_batch(
+                model,
+                start,
+                proposed_key_values,
+                proposed_values_map,
+                create_values,
+                update_values,
+                fts_values,
+                status,
+            )
+            start += FILTER_BATCH_SIZE
+
+        self._finish_query_missing(model, create_values, CREATE_FKS, title)
+        self._finish_query_missing(model, update_values, UPDATE_FKS, title)
+        status.subtitle = ""
+        return num_all_proposed_values
+
+    def _query_missing_model(self, model: type[BaseModel], status: Status):
+        """Find missing model and update create and update sets."""
+        count = 0
+        proposed_values = self.metadata[QUERY_MODELS].pop(model, None)
+        if not proposed_values:
+            return count
+
+        # Finally run the query and get only the correct create_objs
+        return self.query_missing_models(
+            model,
+            proposed_values,
+            status,
+        )
+
+    def _set_fk_totals(self, fk_key: str, status_class):
+        total_fks = 0
+        fks = self.metadata[fk_key]
+        for rows in fks.values():
+            total_fks += len(rows)
+        status = status_class(None, total_fks)
+        self.status_controller.update(status, notify=False)
+        self.metadata[fk_key][TOTAL] = total_fks
+
+    def query_all_missing_models(self):
+        """Find all missing foreign key models."""
+        num_models = self.sum_ops(QUERY_MODELS)
+        status = ImporterQueryMissingTagsStatus(0, num_models)
+        try:
+            if not num_models:
+                return num_models
+            self.status_controller.start(status)
+            for model in tuple(self.metadata[QUERY_MODELS].keys()):
+                if self.abort_event.is_set():
+                    return num_models
+                self._query_missing_model(model, status)
+            self.metadata.pop(QUERY_MODELS)
+            self._set_fk_totals(CREATE_FKS, ImporterCreateTagsStatus)
+            self._set_fk_totals(UPDATE_FKS, ImporterUpdateTagsStatus)
+        finally:
+            self.status_controller.finish(status)
+        return status.complete

codex/librarian/scribe/importer/query/links.py

@@ -0,0 +1,29 @@
+"""Prune link actions."""
+
+from codex.librarian.scribe.importer.const import (
+    LINK_FKS,
+    LINK_M2MS,
+)
+from codex.librarian.scribe.importer.query.links_m2m import QueryPruneLinksM2M
+from codex.librarian.scribe.importer.statii.query import ImporterQueryTagLinksStatus
+
+
+class QueryPruneLinks(QueryPruneLinksM2M):
+    """Prune link actions."""
+
+    def query_prune_comic_links(self):
+        """Prune links that don't need updating."""
+        total_query_ops = self.sum_path_ops(LINK_FKS) + self.sum_path_ops(LINK_M2MS)
+        status = ImporterQueryTagLinksStatus(0, total_query_ops)
+        try:
+            if not total_query_ops:
+                return
+            self.status_controller.start(status)
+            if self.abort_event.is_set():
+                return
+            self.query_prune_comic_fk_links(status)
+            if self.abort_event.is_set():
+                return
+            self.query_prune_comic_m2m_links(status)
+        finally:
+            self.status_controller.finish(status)

codex/librarian/scribe/importer/query/links_fk.py

@@ -0,0 +1,99 @@
+"""Prune M2O links that don't need updating."""
+
+from codex.librarian.scribe.importer.const import (
+    COMIC_FK_FIELD_NAMES,
+    FIELD_NAME_KEYS_REL_MAP,
+    LINK_FKS,
+    PATH_FIELD_NAME,
+)
+from codex.librarian.scribe.importer.query.update_comics import QueryUpdateComics
+from codex.models.comic import Comic
+from codex.settings import LINK_FK_BATCH_SIZE
+
+_QUERY_LINK_FK_PRUNE_ONLY = (
+    PATH_FIELD_NAME,
+    *COMIC_FK_FIELD_NAMES,
+)
+
+
+class QueryPruneLinksFKs(QueryUpdateComics):
+    """Prune M2O links that don't need updating."""
+
+    def pop_links_to_fts(self, path, field_name):
+        """Pop a link to the FTS structure."""
+        link_key = self.metadata[LINK_FKS][path].pop(field_name)
+        self.add_links_to_fts(path, field_name, (link_key,))
+
+    def _query_prune_comic_fk_links_protagonist(
+        self, comic: Comic, path: str, field_name: str, key_values: tuple
+    ):
+        prot = key_values[0]
+        for obj in (comic.main_character, comic.main_team):
+            if obj and obj.name == prot:
+                self.metadata[LINK_FKS][path].pop(field_name)
+                break
+
+    @staticmethod
+    def _query_prune_comic_fk_links_key_equal(field_obj, key_rel, key_value):
+        parts = key_rel.split("__")
+        rel_obj = field_obj
+        key_val = None
+        while parts:
+            key_val = getattr(rel_obj, parts.pop(0), None)
+            rel_obj = key_val
+        return key_val == key_value
+
+    def _query_prune_comic_fk_links_field(self, comic, path, field_name):
+        link_dict = self.metadata[LINK_FKS].get(path)
+        key_values = link_dict[field_name]
+        if field_name == "protagonist":
+            self._query_prune_comic_fk_links_protagonist(
+                comic, path, field_name, key_values
+            )
+            return
+        key_rels = FIELD_NAME_KEYS_REL_MAP[field_name]
+        keys_equal = True
+        field_obj = getattr(comic, field_name)
+        for key_rel, key_value in zip(key_rels, key_values, strict=True):
+            keys_equal = self._query_prune_comic_fk_links_key_equal(
+                field_obj, key_rel, key_value
+            )
+            if not keys_equal:
+                break
+        if keys_equal:
+            self.metadata[LINK_FKS][path].pop(field_name)
+
+    def _query_prune_comic_fk_links_comic(self, comic, status):
+        path = comic.path
+        field_names = tuple(self.metadata[LINK_FKS].get(path).keys())
+        for field_name in field_names:
+            self._query_prune_comic_fk_links_field(comic, path, field_name)
+            status.increment_complete()
+            self.status_controller.update(status)
+        if not self.metadata[LINK_FKS][path]:
+            del self.metadata[LINK_FKS][path]
+
+    def _query_prune_comic_fk_links_batch(self, batch_paths: tuple[str, ...], status):
+        comics = (
+            Comic.objects.filter(library=self.library, path__in=batch_paths)
+            .select_related(*COMIC_FK_FIELD_NAMES)
+            .only(*_QUERY_LINK_FK_PRUNE_ONLY)
+        )
+        for comic in comics:
+            self._query_prune_comic_fk_links_comic(comic, status)
+
+    def query_prune_comic_fk_links(self, status):
+        """Prune comic fk links that already exist."""
+        status.subtitle = "Many to One"
+        self.status_controller.update(status)
+        paths = tuple(self.metadata[LINK_FKS].keys())
+        num_paths = len(paths)
+
+        start = 0
+        while start < num_paths:
+            if self.abort_event.is_set():
+                return
+            end = start + LINK_FK_BATCH_SIZE
+            batch_paths = paths[start:end]
+            self._query_prune_comic_fk_links_batch(batch_paths, status)
+            start += LINK_FK_BATCH_SIZE

codex/librarian/scribe/importer/query/links_m2m.py

@@ -0,0 +1,102 @@
+"""Prune M2M links that don't need updating."""
+
+from codex.librarian.scribe.importer.const import (
+    COMIC_M2M_FIELD_NAMES,
+    DELETE_M2MS,
+    FIELD_NAME_KEY_ATTRS_MAP,
+    LINK_M2MS,
+)
+from codex.librarian.scribe.importer.query.links_fk import QueryPruneLinksFKs
+from codex.models.base import BaseModel, NamedModel
+from codex.models.comic import Comic
+from codex.models.groups import BrowserGroupModel
+from codex.settings import LINK_M2M_BATCH_SIZE
+
+
+class QueryPruneLinksM2M(QueryPruneLinksFKs):
+    """Prune M2M links that don't need updating."""
+
+    @staticmethod
+    def _m2m_obj_to_key_tuple(key_attrs: tuple[str, ...], m2m_obj: BaseModel):
+        """Create a key value tuple from a db obj."""
+        value_tuple = []
+        for attr in key_attrs:
+            value = getattr(m2m_obj, attr)
+            if isinstance(value, NamedModel | BrowserGroupModel):
+                value = value.name
+            value_tuple.append(value)
+        return tuple(value_tuple)
+
+    def _query_prune_comic_m2m_links_field_obj(
+        self,
+        comic: Comic,
+        field_name: str,
+        m2m_obj: BaseModel,
+        kept_existing_values: set,
+        proposed_values: set,
+    ):
+        """Remove existing m2m links from the action list and add missing ones delete list."""
+        # transform objs into tuples
+        key_attrs = FIELD_NAME_KEY_ATTRS_MAP[field_name]
+        existing_key_value_tuple = self._m2m_obj_to_key_tuple(key_attrs, m2m_obj)
+        if existing_key_value_tuple in proposed_values:
+            # If already linked correctly, remove from action set
+            kept_existing_values.add(existing_key_value_tuple)
+            proposed_values.discard(existing_key_value_tuple)
+            deleted = False
+        else:
+            # Delete existing m2m links that aren't in the new comic
+            delete_m2m = (comic.pk, m2m_obj.pk)
+            self.metadata[DELETE_M2MS][field_name].add(delete_m2m)
+            deleted = True
+        return deleted
+
+    def _query_prune_comic_m2m_links_field(self, comic: Comic, field_name: str, status):
+        if field_name not in self.metadata[DELETE_M2MS]:
+            self.metadata[DELETE_M2MS][field_name] = set()
+
+        m2m_objs = getattr(comic, field_name).all()
+        deleted = False
+        kept_existing_values = set()
+        proposed_values = self.metadata[LINK_M2MS][comic.path][field_name]
+        for m2m_obj in m2m_objs:
+            deleted |= self._query_prune_comic_m2m_links_field_obj(
+                comic, field_name, m2m_obj, kept_existing_values, proposed_values
+            )
+            status.increment_complete()
+            self.status_controller.update(status)
+        if kept_existing_values and (deleted or proposed_values):
+            self.add_to_fts_existing(comic.pk, field_name, tuple(kept_existing_values))
+        if not self.metadata[LINK_M2MS][comic.path][field_name]:
+            del self.metadata[LINK_M2MS][comic.path][field_name]
+
+    def _query_prune_comic_m2m_links_comic(self, comic: Comic, status):
+        field_names = tuple(self.metadata[LINK_M2MS][comic.path].keys())
+        for field_name in field_names:
+            self._query_prune_comic_m2m_links_field(comic, field_name, status)
+        if not self.metadata[LINK_M2MS][comic.path]:
+            del self.metadata[LINK_M2MS][comic.path]
+
+    def _query_prune_comic_m2m_links_batch(self, paths: tuple[str], status):
+        comics = (
+            Comic.objects.filter(library=self.library, path__in=paths)
+            # prefetching deep links means a batch size of 190 or less
+            .prefetch_related(*COMIC_M2M_FIELD_NAMES)
+            .only(*COMIC_M2M_FIELD_NAMES)
+        )
+        for comic in comics.iterator(chunk_size=LINK_M2M_BATCH_SIZE):
+            self._query_prune_comic_m2m_links_comic(comic, status)
+
+    def query_prune_comic_m2m_links(self, status):
+        """Prune comic m2m links that already exists or should be deleted."""
+        status.subtitle = "Many to Many"
+        self.status_controller.update(status)
+        paths = tuple(self.metadata[LINK_M2MS].keys())
+        self._query_prune_comic_m2m_links_batch(paths, status)
+        field_names = tuple(self.metadata[DELETE_M2MS].keys())
+        for field_name in field_names:
+            rows = self.metadata[DELETE_M2MS][field_name]
+            if not rows:
+                del self.metadata[DELETE_M2MS][field_name]
+        status.increment_complete(len(field_names))
+        self.status_controller.update(status)

codex/librarian/scribe/importer/query/update_comics.py

@@ -0,0 +1,59 @@
+"""Move comics that need only updating into correct structure."""
+
+from codex.librarian.scribe.importer.const import (
+    BULK_UPDATE_COMIC_FIELDS,
+    CREATE_COMICS,
+    UPDATE_COMICS,
+)
+from codex.librarian.scribe.importer.query.foreign_keys import (
+    QueryForeignKeysQueryImporter,
+)
+from codex.librarian.scribe.importer.statii.create import (
+    ImporterCreateComicsStatus,
+    ImporterUpdateComicsStatus,
+)
+from codex.librarian.scribe.importer.statii.query import ImporterQueryComicUpdatesStatus
+from codex.models import Comic
+
+
+class QueryUpdateComics(QueryForeignKeysQueryImporter):
+    """Move comics that need only updating into correct structure."""
+
+    def query_update_comics(self):
+        """Pop existing comics from create & move to update if needed."""
+        paths = tuple(self.metadata[CREATE_COMICS].keys())
+        status = ImporterQueryComicUpdatesStatus(0, len(paths))
+        try:
+            if not paths:
+                return
+            self.status_controller.start(status)
+
+            comics = Comic.objects.filter(library=self.library, path__in=paths).only(
+                *BULK_UPDATE_COMIC_FIELDS
+            )
+            status.increment_complete(len(paths) - comics.count())
+
+            for comic in comics:
+                if self.abort_event.is_set():
+                    return
+                proposed_comic_dict = self.metadata[CREATE_COMICS].pop(comic.path)
+                update_comic_dict = {
+                    key: value
+                    for key, value in proposed_comic_dict.items()
+                    if getattr(comic, key) != value
+                }
+                if update_comic_dict:
+                    self.metadata[UPDATE_COMICS][comic.pk] = update_comic_dict
+                status.increment_complete()
+                self.status_controller.update(status)
+
+            create_status = ImporterCreateComicsStatus(
+                None, len(self.metadata[CREATE_COMICS])
+            )
+            self.status_controller.update(create_status)
+            update_status = ImporterUpdateComicsStatus(
+                None, len(self.metadata[UPDATE_COMICS])
+            )
+            self.status_controller.update(update_status)
+        finally:
+            self.status_controller.finish(status)