codex 1.6.0__py3-none-any.whl → 1.6.0a0__py3-none-any.whl

codex/librarian/importer/cache.py

@@ -1,93 +0,0 @@
-"""Update Groups timestamp for cover cache busting."""
-
-from django.db.models.aggregates import Count
-from django.db.models.functions.datetime import Now
-from django.db.models.query import Q
-
-from codex.librarian.importer.init import InitImporter
-from codex.librarian.importer.status import ImportStatusTypes
-from codex.models import StoryArc, Volume
-from codex.status import Status
-from codex.views.const import GROUP_MODELS
-
-_UPDATE_FIELDS = ("updated_at",)
-
-
-class CacheUpdateImporter(InitImporter):
-    """Update Groups timestamp for cover cache busting."""
-
-    @staticmethod
-    def _get_update_filter(model, start_time, force_update_group_map):
-        # Get groups with comics updated during this import
-        rel = "storyarcnumber__" if model == StoryArc else ""
-        updated_at_rel = rel + "comic__updated_at__gt"
-        updated_filter = {updated_at_rel: start_time}
-        update_filter = Q(**updated_filter)
-
-        # Get groups with custom covers updated during this import
-        if model != Volume:
-            update_filter |= Q(custom_cover__updated_at__gt=start_time)
-
-        # Get groups to be force updated (usually those with deleted children)
-        if pks := force_update_group_map.get(model):
-            update_filter |= Q(pk__in=pks)
-
-        return update_filter
-
-    @staticmethod
-    def _add_child_count_filter(qs, model):
-        """Filter out groups with no comics."""
-        rel_prefix = "storyarcnumber__" if model == StoryArc else ""
-        rel_prefix += "comic"
-        qs = qs.alias(child_count=Count(f"{rel_prefix}__pk", distinct=True))
-        return qs.filter(child_count__gt=0)
-
-    @classmethod
-    def _update_group_model(cls, force_update_group_map, model, start_time, log_list):
-        """Update a single group model."""
-        update_filter = cls._get_update_filter(
-            model, start_time, force_update_group_map
-        )
-        qs = model.objects.filter(update_filter)
-        qs = cls._add_child_count_filter(qs, model)
-
-        qs = qs.distinct()
-        qs = qs.only(*_UPDATE_FIELDS)
-
-        updated = []
-        for obj in qs:
-            obj.updated_at = Now()
-            updated.append(obj)
-
-        count = len(updated)
-        if count:
-            model.objects.bulk_update(updated, _UPDATE_FIELDS)
-            log_list.append(f"{count} {model.__name__}s")
-        return count
-
-    def update_all_groups(self, force_update_group_map, start_time):
-        """Update timestamps for each group for cover cache busting."""
-        total_count = 0
-        status = Status(ImportStatusTypes.GROUP_UPDATE)
-        self.status_controller.start(status)
-        try:
-            log_list = []
-            for model in GROUP_MODELS:
-                # self.log.debug(f"Updating timestamps for {model.__name__}s...")
-                count = self._update_group_model(
-                    force_update_group_map, model, start_time, log_list
-                )
-                if count:
-                    self.log.debug(f"Updated {count} {model.__name__}s timestamps.")
-                status.add_complete(count)
-                self.status_controller.update(status, notify=False)
-                total_count += count
-
-            if total_count:
-                groups_log = ", ".join(log_list)
-                self.log.info(  # type: ignore
-                    f"Updated timestamps for {groups_log}."
-                )
-                self.changed += total_count
-        finally:
-            self.status_controller.finish(status)

codex/librarian/importer/create_covers.py

@@ -1,96 +0,0 @@
-"""Create Custom Covers."""
-
-from django.db.models.functions.datetime import Now
-
-from codex.librarian.importer.const import (
-    COVERS_CREATE,
-    COVERS_UPDATE,
-    CUSTOM_COVER_UPDATE_FIELDS,
-    LINK_COVER_PKS,
-)
-from codex.librarian.importer.create_comics import CreateComicsImporter
-from codex.librarian.importer.status import ImportStatusTypes
-from codex.models import (
-    CustomCover,
-)
-from codex.status import Status
-
-
-class CreateCoversImporter(CreateComicsImporter):
-    """Create Custom Covers."""
-
-    def update_custom_covers(self, status=None):
-        """Update Custom Covers."""
-        update_covers_qs = self.metadata.pop(COVERS_UPDATE, None)
-        if not update_covers_qs:
-            return
-        update_covers_count = update_covers_qs.count()
-        if not update_covers_count:
-            return
-        status = Status(ImportStatusTypes.COVERS_MODIFIED, 0, update_covers_count)
-        self.status_controller.start(status)
-
-        update_covers = []
-        for cover in update_covers_qs.only(*CUSTOM_COVER_UPDATE_FIELDS):
-            cover.updated_at = Now()
-            cover.presave()
-            update_covers.append(cover)
-
-        count = 0
-        if update_covers:
-            CustomCover.objects.bulk_update(update_covers, CUSTOM_COVER_UPDATE_FIELDS)
-            update_cover_pks = update_covers_qs.values_list("pk", flat=True)
-            if LINK_COVER_PKS not in self.metadata:
-                self.metadata[LINK_COVER_PKS] = set()
-            self.metadata[LINK_COVER_PKS].update(update_cover_pks)
-            self._remove_covers(update_cover_pks, custom=True)  # type: ignore
-            count = len(update_covers)
-            if status:
-                status.add_complete(count)
-
-            link_covers_status = Status(
-                ImportStatusTypes.COVERS_LINK, 0, len(self.metadata[LINK_COVER_PKS])
-            )
-            self.status_controller.update(link_covers_status, notify=False)
-
-        self.changed += count
-        self.status_controller.finish(status)
-
-    def create_custom_covers(self, status=None):
-        """Create Custom Covers."""
-        create_cover_paths = self.metadata.pop(COVERS_CREATE, ())
-        num_create_cover_paths = len(create_cover_paths)
-        if not num_create_cover_paths:
-            return
-        status = Status(ImportStatusTypes.COVERS_MODIFIED, 0, num_create_cover_paths)
-        self.status_controller.start(status)
-
-        create_covers = []
-        for path in create_cover_paths:
-            cover = CustomCover(library=self.library, path=path)
-            cover.presave()
-            create_covers.append(cover)
-
-        count = 0
-        if create_covers:
-            objs = CustomCover.objects.bulk_create(
-                create_covers,
-                update_conflicts=True,
-                update_fields=("path", "stat"),
-                unique_fields=CustomCover._meta.unique_together[0],
-            )
-            created_pks = frozenset(obj.pk for obj in objs)
-            if LINK_COVER_PKS not in self.metadata:
-                self.metadata[LINK_COVER_PKS] = set()
-            self.metadata[LINK_COVER_PKS].update(created_pks)
-            count = len(created_pks)
-            if status:
-                status.add_complete(count)
-
-            link_covers_status = Status(
-                ImportStatusTypes.COVERS_LINK, 0, len(self.metadata[LINK_COVER_PKS])
-            )
-            self.status_controller.update(link_covers_status, notify=False)
-
-        self.changed += count
-        self.status_controller.finish(status)

codex/librarian/importer/importer.py

@@ -1,96 +0,0 @@
-"""The main importer class."""
-
-from time import time
-
-from django.core.cache import cache
-from humanize import naturaldelta
-
-from codex.librarian.importer.moved import MovedImporter
-from codex.librarian.importer.status import ImportStatusTypes
-from codex.librarian.notifier.tasks import FAILED_IMPORTS_TASK, LIBRARY_CHANGED_TASK
-from codex.librarian.search.tasks import SearchIndexUpdateTask
-from codex.librarian.tasks import DelayedTasks
-
-
-class ComicImporter(MovedImporter):
-    """Initialize, run and finish a bulk import."""
-
-    ##########
-    # FINISH #
-    ##########
-    def _finish_apply_status(self):
-        """Finish all librarian statuses."""
-        self.library.update_in_progress = False
-        self.library.save()
-        self.status_controller.finish_many(ImportStatusTypes.values)
-
-    def _finish_apply(self, new_failed_imports, imported_count):
-        """Perform final tasks when the apply is done."""
-        if self.changed:
-            cache.clear()
-            elapsed_time = time() - self.start_time.timestamp()
-            elapsed = naturaldelta(elapsed_time)
-            log_txt = f"Updated library {self.library.path} in {elapsed}."
-            if imported_count:
-                cps = round(imported_count / elapsed_time, 1)
-                log_txt += (
-                    f" Imported {imported_count} comics at {cps} comics per second."
-                )
-            else:
-                log_txt += " No comics to import."
-            self.log.info(log_txt)
-            self.librarian_queue.put(LIBRARY_CHANGED_TASK)
-
-            # Wait to start the search index update in case more updates are incoming.
-            until = time() + 3
-            delayed_search_task = DelayedTasks(until, (SearchIndexUpdateTask(False),))
-            self.librarian_queue.put(delayed_search_task)
-        else:
-            self.log.info("No updates neccissary.")
-        if new_failed_imports:
-            self.librarian_queue.put(FAILED_IMPORTS_TASK)
-
-    def apply(self):
-        """Bulk import comics."""
-        try:
-            self.init_apply()
-            self.move_and_modify_dirs()
-
-            #############
-            # AGGREGATE #
-            #############
-            self.get_aggregate_metadata()
-
-            #########
-            # QUERY #
-            #########
-            self.query_all_missing_fks()
-            self.query_missing_custom_covers()
-
-            #####################
-            # UPDATE AND CREATE #
-            #####################
-            self.create_all_fks()
-            self.update_custom_covers()
-            self.create_custom_covers()
-
-            imported_count = self.bulk_update_comics()
-            imported_count += self.bulk_create_comics()
-
-            ########
-            # LINK #
-            ########
-            self.bulk_query_and_link_comic_m2m_fields()
-            self.link_custom_covers()
-            self.changed += imported_count
-
-            new_failed_imports = self.fail_imports()
-
-            deleted_comic_groups = self.delete()
-
-            self.update_all_groups(deleted_comic_groups, self.start_time)
-
-        finally:
-            self._finish_apply_status()
-
-        self._finish_apply(new_failed_imports, imported_count)

codex/librarian/importer/init.py

@@ -1,265 +0,0 @@
-"""Initiale Importer."""
-
-import logging
-from pathlib import Path
-from time import sleep, time
-from typing import Any
-
-from django.utils.timezone import now
-
-from codex.librarian.importer.status import ImportStatusTypes
-from codex.librarian.importer.tasks import ImportDBDiffTask
-from codex.librarian.search.status import SearchIndexStatusTypes
-from codex.librarian.search.tasks import SearchIndexAbortTask
-from codex.models import Library
-from codex.status import Status
-from codex.worker_base import WorkerBaseMixin
-
-_WRITE_WAIT_EXPIRY = 60
-
-
-class InitImporter(WorkerBaseMixin):
-    """Initiale Importer."""
-
-    def __init__(self, task: ImportDBDiffTask, log_queue, librarian_queue):
-        """Initialize the import."""
-        self.init_worker(log_queue, librarian_queue)
-        self.task: ImportDBDiffTask = task
-        self.metadata: dict[str, Any] = {}
-        self.changed: int = 0
-        self.library = Library.objects.only("path", "update_in_progress").get(
-            pk=self.task.library_id
-        )
-
-    def _wait_for_filesystem_ops_to_finish(self) -> bool:
-        """Watchdog sends events before filesystem events finish, so wait for them."""
-        started_checking = time()
-
-        # Don't wait for deletes to complete.
-        # Do wait for move, modified, create files before import.
-        all_modified_paths = (
-            frozenset(self.task.dirs_moved.values())
-            | frozenset(self.task.files_moved.values())
-            | self.task.dirs_modified
-            | self.task.files_modified
-            | self.task.files_created
-        )
-
-        old_total_size = -1
-        total_size = 0
-        wait_time = 2
-        while old_total_size != total_size:
-            if old_total_size > 0:
-                # second time around or more
-                sleep(wait_time)
-                wait_time = wait_time**2
-                self.log.debug(
-                    f"Waiting for files to copy before import: "
-                    f"{old_total_size} != {total_size}"
-                )
-            if time() - started_checking > _WRITE_WAIT_EXPIRY:
-                return True
-
-            old_total_size = total_size
-            total_size = 0
-            for path_str in all_modified_paths:
-                path = Path(path_str)
-                if path.exists():
-                    total_size += Path(path).stat().st_size
-        return False
-
-    #######
-    # LOG #
-    #######
-    def _log_task_construct_dirs_log(self):
-        """Construct dirs log line."""
-        dirs_log = []
-        if self.task.dirs_moved:
-            dirs_log += [f"{len(self.task.dirs_moved)} moved"]
-        if self.task.dirs_modified:
-            dirs_log += [f"{len(self.task.dirs_modified)} modified"]
-        if self.task.dirs_deleted:
-            dirs_log += [f"{len(self.task.dirs_deleted)} deleted"]
-        return dirs_log
-
-    def _log_task_construct_comics_log(self):
-        """Construct comcis log line."""
-        comics_log = []
-        if self.task.files_moved:
-            comics_log += [f"{len(self.task.files_moved)} moved"]
-        if self.task.files_modified:
-            comics_log += [f"{len(self.task.files_modified)} modified"]
-        if self.task.files_created:
-            comics_log += [f"{len(self.task.files_created)} created"]
-        if self.task.files_deleted:
-            comics_log += [f"{len(self.task.files_deleted)} deleted"]
-        return comics_log
-
-    def _log_task(self):
-        """Log the watchdog event self.task."""
-        if self.log.getEffectiveLevel() < logging.DEBUG:
-            return
-
-        self.log.debug(f"Updating library {self.library.path}...")
-        comics_log = self._log_task_construct_comics_log()
-        if comics_log:
-            log = "Comics: "
-            log += ", ".join(comics_log)
-            self.log.debug("  " + log)
-
-        dirs_log = self._log_task_construct_dirs_log()
-        if dirs_log:
-            log = "Folders: "
-            log += ", ".join(dirs_log)
-            self.log.debug("  " + log)
-
-    ########
-    # INIT #
-    ########
-    def _init_librarian_status_moved(self, status_list):
-        """Initialize moved statuses."""
-        search_index_updates = 0
-        if self.task.dirs_moved:
-            status_list += [
-                Status(ImportStatusTypes.DIRS_MOVED, None, len(self.task.dirs_moved))
-            ]
-        if self.task.files_moved:
-            status_list += [
-                Status(ImportStatusTypes.FILES_MOVED, None, len(self.task.files_moved))
-            ]
-            search_index_updates += len(self.task.files_moved)
-        if self.task.covers_moved:
-            status_list += [
-                Status(
-                    ImportStatusTypes.COVERS_MOVED, None, len(self.task.covers_moved)
-                )
-            ]
-        if self.task.dirs_modified:
-            status_list += [
-                Status(
-                    ImportStatusTypes.DIRS_MODIFIED, None, len(self.task.dirs_modified)
-                )
-            ]
-        return search_index_updates
-
-    def _init_if_modified_or_created(self, path, status_list):
-        """Initialize librarian statuses for modified or created ops."""
-        total_paths = len(self.task.files_modified) + len(self.task.files_created)
-        status_list += [
-            Status(ImportStatusTypes.AGGREGATE_TAGS, 0, total_paths, subtitle=path),
-            Status(ImportStatusTypes.QUERY_MISSING_FKS),
-            Status(ImportStatusTypes.QUERY_MISSING_COVERS),
-            Status(ImportStatusTypes.CREATE_FKS),
-        ]
-        if self.task.files_modified:
-            status_list += [
-                Status(
-                    ImportStatusTypes.FILES_MODIFIED,
-                    None,
-                    len(self.task.files_modified),
-                )
-            ]
-        if self.task.covers_modified:
-            status_list += [
-                Status(
-                    ImportStatusTypes.COVERS_MODIFIED,
-                    None,
-                    len(self.task.covers_modified),
-                )
-            ]
-
-        if self.task.files_created or self.task.covers_created:
-            status_list += [
-                Status(
-                    ImportStatusTypes.FILES_CREATED, None, len(self.task.files_created)
-                )
-            ]
-        if self.task.covers_created:
-            status_list += [
-                Status(
-                    ImportStatusTypes.COVERS_CREATED,
-                    None,
-                    len(self.task.covers_created),
-                )
-            ]
-
-        if self.task.files_modified or self.task.files_created:
-            status_list += [Status(ImportStatusTypes.LINK_M2M_FIELDS)]
-
-        num_covers_linked = (
-            len(self.task.covers_moved)
-            + len(self.task.covers_modified)
-            + len(self.task.covers_created)
-        )
-        if num_covers_linked:
-            status_list += [
-                Status(ImportStatusTypes.COVERS_LINK, None, num_covers_linked)
-            ]
-
-        return total_paths
-
-    def _init_librarian_status_deleted(self, status_list):
-        """Init deleted statuses."""
-        search_index_updates = 0
-        if self.task.files_deleted:
-            status_list += [
-                Status(
-                    ImportStatusTypes.FILES_DELETED, None, len(self.task.files_deleted)
-                )
-            ]
-            search_index_updates += len(self.task.files_deleted)
-        if self.task.covers_deleted:
-            status_list += [
-                Status(
-                    ImportStatusTypes.COVERS_DELETED,
-                    None,
-                    len(self.task.covers_deleted),
-                )
-            ]
-        return search_index_updates
-
-    @staticmethod
-    def _init_librarian_status_search_index(search_index_updates, status_list):
-        """Init search index statuses."""
-        status_list += [
-            Status(
-                SearchIndexStatusTypes.SEARCH_INDEX_UPDATE,
-                0,
-                search_index_updates,
-            ),
-            Status(SearchIndexStatusTypes.SEARCH_INDEX_REMOVE),
-        ]
-
-    def _init_librarian_status(self, path):
-        """Update the librarian status self.tasks."""
-        status_list = []
-        search_index_updates = 0
-        search_index_updates += self._init_librarian_status_moved(status_list)
-        if (
-            self.task.files_modified
-            or self.task.files_created
-            or self.task.covers_modified
-            or self.task.covers_created
-        ):
-            search_index_updates += self._init_if_modified_or_created(path, status_list)
-        search_index_updates += self._init_librarian_status_deleted(status_list)
-        status_list += [Status(ImportStatusTypes.GROUP_UPDATE)]
-        self._init_librarian_status_search_index(search_index_updates, status_list)
-        self.status_controller.start_many(status_list)
-
-    def init_apply(self):
-        """Initialize the library and status flags."""
-        self.start_time = now()
-        self.librarian_queue.put(SearchIndexAbortTask())
-        self.library.update_in_progress = True
-        self.library.save()
-        too_long = self._wait_for_filesystem_ops_to_finish()
-        if too_long:
-            self.log.warning(
-                "Import apply waited for the filesystem to stop changing too long. "
-                "Try polling again once files have finished copying"
-                f" in library: {self.library.path}"
-            )
-            return
-        self._log_task()
-        self._init_librarian_status(self.library.path)

codex/librarian/importer/link_covers.py

@@ -1,73 +0,0 @@
-"""Link Covers."""
-
-from pathlib import Path
-
-from codex.librarian.importer.const import CLASS_CUSTOM_COVER_GROUP_MAP, LINK_COVER_PKS
-from codex.librarian.importer.failed_imports import FailedImportsImporter
-from codex.librarian.importer.status import ImportStatusTypes
-from codex.models import CustomCover, Folder
-from codex.status import Status
-
-
-class LinkCoversImporter(FailedImportsImporter):
-    """Link Covers methods."""
-
-    def _link_custom_cover_prepare(self, cover, model_map):
-        """Prepare one cover in the model map for bulk update."""
-        if cover.library and cover.library.covers_only:
-            model = CLASS_CUSTOM_COVER_GROUP_MAP.inverse.get(cover.group)
-            if not model:
-                self.log.warning(f"Custom Cover model not found for {cover.path}")
-                return
-            group_filter = {"sort_name": cover.sort_name}
-        else:
-            model = Folder
-            path = str(Path(cover.path).parent)
-            group_filter = {"path": path}
-        qs = model.objects.filter(**group_filter).exclude(custom_cover=cover)
-        if not qs.exists():
-            return
-
-        if model not in model_map:
-            model_map[model] = []
-
-        for obj in qs.iterator():
-            obj.custom_cover = cover
-            model_map[model].append(obj)
-
-    def _link_custom_cover_group(self, model, objs, status):
-        """Bulk link a group to it's custom covers."""
-        count = 0
-        if not objs:
-            return count
-        model.objects.bulk_update(objs, ["custom_cover"])
-        count += len(objs)
-        self.log.info(f"Linked {count} custom covers to {model.__name__}s")
-        status.complete += count
-        self.status_controller.update(status, notify=False)
-        return count
-
-    def link_custom_covers(self):
-        """Link Custom Covers to Groups."""
-        link_cover_pks = self.metadata.get(LINK_COVER_PKS, {})
-        num_link_cover_pks = len(link_cover_pks)
-        if not num_link_cover_pks:
-            return 0
-        status = Status(ImportStatusTypes.COVERS_LINK, 0, num_link_cover_pks)
-        try:
-            self.status_controller.start(status)
-            # Aggregate objs to update for each group model.
-            model_map = {}
-            covers = CustomCover.objects.filter(pk__in=link_cover_pks).only(
-                "library", "path"
-            )
-            for cover in covers:
-                self._link_custom_cover_prepare(cover, model_map)
-
-            # Bulk update each model type
-            total_count = 0
-            for model, objs in model_map.items():
-                total_count += self._link_custom_cover_group(model, objs, status)
-        finally:
-            self.status_controller.finish(status)
-        return total_count

codex/librarian/importer/query_covers.py

@@ -1,52 +0,0 @@
-"""Query Missing Custom Covers."""
-
-from codex.librarian.importer.const import (
-    COVERS_CREATE,
-    COVERS_UPDATE,
-)
-from codex.librarian.importer.create_fks import CreateForeignKeysImporter
-from codex.librarian.importer.status import ImportStatusTypes
-from codex.logger.logging import get_logger
-from codex.models.paths import CustomCover
-from codex.status import Status
-
-LOG = get_logger(__name__)
-
-
-class QueryCustomCoversImporter(CreateForeignKeysImporter):
-    """Query Missing Custom Covers."""
-
-    def query_missing_custom_covers(self):
-        """Identify update & create covers."""
-        cover_paths = self.task.covers_created | self.task.covers_modified
-        num_cover_paths = len(cover_paths)
-        if not num_cover_paths:
-            return
-        self.log.debug(f"Querying {num_cover_paths} custom cover_paths")
-        status = Status(ImportStatusTypes.QUERY_MISSING_COVERS, None, num_cover_paths)
-        self.status_controller.start(status)
-
-        update_covers_qs = CustomCover.objects.filter(
-            library=self.library, path__in=cover_paths
-        )
-        self.metadata[COVERS_UPDATE] = update_covers_qs
-        update_cover_paths = frozenset(update_covers_qs.values_list("path", flat=True))
-        update_count = len(update_cover_paths)
-        update_status = Status(ImportStatusTypes.COVERS_MODIFIED, 0, update_count)
-        self.status_controller.update(update_status, notify=False)
-
-        create_cover_paths = cover_paths - update_cover_paths
-        self.metadata[COVERS_CREATE] = create_cover_paths
-        create_count = len(create_cover_paths)
-        create_status = Status(ImportStatusTypes.COVERS_CREATED, 0, create_count)
-        self.status_controller.update(create_status, notify=False)
-
-        self.task.covers_created = self.task.covers_modified = frozenset()
-
-        count = create_count + update_count
-        if count:
-            self.log.debug(
-                f"Discovered {update_count} custom covers to update and {create_count} to create."
-            )
-        self.status_controller.finish(status)
-        return