codex 1.7.15__py3-none-any.whl → 1.8.0a2__py3-none-any.whl

codex/librarian/importer/extract.py

@@ -1,284 +0,0 @@
-"""Clean metadata before importing."""
-
-from contextlib import suppress
-from decimal import ROUND_DOWN, Decimal
-from html import unescape
-from typing import Any
-from zipfile import BadZipFile
-
-from comicbox.box import Comicbox
-from comicbox.box.computed import IDENTIFIERS_KEY
-from comicbox.exceptions import UnsupportedArchiveTypeError
-from comicbox.schemas.comicbox_mixin import CONTRIBUTORS_KEY
-from django.db.models.fields import (
-    CharField,
-    DecimalField,
-    PositiveSmallIntegerField,
-    TextField,
-)
-from nh3 import clean
-from rarfile import BadRarFile
-
-from codex.librarian.importer.const import FIS, STORY_ARCS_METADATA_KEY
-from codex.librarian.importer.query_fks import QueryForeignKeysImporter
-from codex.models import Comic
-from codex.models.named import (
-    ContributorPerson,
-    ContributorRole,
-    Identifier,
-    IdentifierType,
-    StoryArc,
-)
-
-_MD_INVALID_KEYS = frozenset(
-    (
-        "id",
-        "created_at",
-        "library",
-        "parent_folder",
-        "pk",
-        "stat",
-        "story_arc_number",
-        "updated_at",
-    )
-)
-_MD_VALID_KEYS = tuple(
-    sorted(
-        frozenset(
-            (
-                *(
-                    field.name
-                    for field in Comic._meta.get_fields()
-                    if field.name not in _MD_INVALID_KEYS
-                ),
-                "story_arcs",
-            )
-        )
-    )
-)
-
-
-def _get_keys_by_field_type(field_type, exclude=None):
-    exclude_set = exclude if exclude else set()
-    return tuple(
-        sorted(
-            frozenset(
-                field.name
-                for field in Comic._meta.get_fields()
-                if field not in exclude_set and isinstance(field, field_type)
-            )
-        )
-    )
-
-
-_MD_CHAR_KEYS = _get_keys_by_field_type(CharField, exclude={"path"})
-_MD_TEXT_KEYS = _get_keys_by_field_type(TextField)
-_MD_STR_KEYS = tuple(sorted(_MD_CHAR_KEYS + _MD_TEXT_KEYS))
-_MD_DECIMAL_KEYS = _get_keys_by_field_type(DecimalField)
-_MD_PSI_KEYS = _get_keys_by_field_type(PositiveSmallIntegerField)
-_PSI_MAX = 2**31 - 1
-_SI_MIN = 2**15 * -1
-_SI_MAX = 2**15 - 1
-_DECIMAL_ZERO = Decimal("0.00")
-_ALPHA_2_LEN = 2
-
-
-class ExtractMetadataImporter(QueryForeignKeysImporter):
-    """Clean metadata before importing."""
-
-    @staticmethod
-    def _title_to_name(md):
-        """Convert title to name for comics."""
-        with suppress(KeyError):
-            md["name"] = md.pop("title")
-
-    @staticmethod
-    def _prune_extra_keys(dirty_md: dict[str, Any]) -> dict[str, Any]:
-        """Remove unused keys."""
-        allowed_md = {}
-        for key in _MD_VALID_KEYS:
-            val = dirty_md.get(key)
-            if val is not None:
-                allowed_md[key] = val
-        return allowed_md
-
-    @staticmethod
-    def _assign_or_pop(md, key, value):
-        if value is None:
-            md.pop(key, None)
-        else:
-            md[key] = value
-
-    @classmethod
-    def _clean_decimal(cls, value, field_name: str):
-        field: DecimalField = Comic._meta.get_field(field_name)  # type: ignore[reportAssignmentType]
-        try:
-            quantize_str = Decimal(f"1e-{field.decimal_places}")
-            value = value.quantize(quantize_str, rounding=ROUND_DOWN)
-            decimal_max = Decimal(10 ** (field.max_digits - 2) - 1)
-            value = value.min(decimal_max)
-        except Exception:
-            value = None if field.null else _DECIMAL_ZERO
-        return value
-
-    @classmethod
-    def _clean_comic_decimals(cls, md: dict[str, Any]) -> None:
-        """Clean decimal values."""
-        for key in _MD_DECIMAL_KEYS:
-            value = md.get(key)
-            if value is None:
-                continue
-            value = cls._clean_decimal(value, key)
-            cls._assign_or_pop(md, key, value)
-
-    @staticmethod
-    def _clean_int(md, key, minimum, maximum):
-        try:
-            value = md.get(key)
-            if value is not None:
-                value = min(value, maximum)
-                value = max(value, minimum)
-        except Exception:
-            field = Comic._meta.get_field(key)
-            value = None if field.null else 0
-        return value
-
-    @classmethod
-    def _clean_comic_positive_small_ints(
-        cls,
-        md: dict[str, Any],
-    ) -> None:
-        """Clean positive small integers."""
-        for key in _MD_PSI_KEYS:
-            value = cls._clean_int(md, key, 0, _PSI_MAX)
-            cls._assign_or_pop(md, key, value)
-
-    @classmethod
-    def _clean_comic_small_ints(cls, md: dict[str, Any]):
-        # clean sub value
-        obj = md.get("volume", {})
-        key = "name"
-        value = cls._clean_int(obj, key, _SI_MIN, _SI_MAX)
-        cls._assign_or_pop(obj, key, value)
-
-    @staticmethod
-    def _clean_strfield(model, field_name, value):
-        try:
-            if value:
-                value = value.strip()
-                field = model._meta.get_field(field_name)  # type: ignore[reportAssignmentType]
-                max_length = getattr(field, "max_length", 0)
-                if max_length:
-                    value = value[:max_length]
-                value = clean(value)
-                value = unescape(value)
-            if not value:
-                value = None
-        except Exception:
-            value = None
-        return value
-
-    @classmethod
-    def _clean_strfields(cls, md: dict[str, Any]) -> None:
-        for key in _MD_STR_KEYS:
-            value = md.get(key)
-            value = cls._clean_strfield(Comic, key, value)
-            cls._assign_or_pop(md, key, value)
-
-    @classmethod
-    def _clean_pycountry(cls, md, key, lookup) -> None:
-        try:
-            value = md.get(key)
-            if value and len(value) > _ALPHA_2_LEN and (obj := lookup.lookup(value)):
-                value = obj.alpha_2
-            if not value:
-                value = None
-        except Exception:
-            value = None
-        cls._assign_or_pop(md, key, value)
-
-    @classmethod
-    def _clean_contributors(cls, md):
-        contributors = md.get(CONTRIBUTORS_KEY)
-        if not contributors:
-            return
-        clean_contributors = {}
-        for role, persons in contributors.items():
-            clean_role = cls._clean_strfield(ContributorRole, "name", role)
-            clean_persons = set()
-            for person in persons:
-                clean_person = cls._clean_strfield(ContributorPerson, "name", person)
-                if clean_person:
-                    clean_persons.add(clean_person)
-            clean_contributors[clean_role] = clean_persons
-        cls._assign_or_pop(md, CONTRIBUTORS_KEY, clean_contributors)
-
-    @classmethod
-    def _clean_story_arcs(cls, md):
-        story_arcs = md.get(STORY_ARCS_METADATA_KEY)
-        if not story_arcs:
-            return
-        clean_story_arcs = {}
-        for story_arc, number in story_arcs.items():
-            clean_story_arc = cls._clean_strfield(StoryArc, "name", story_arc)
-            if clean_story_arc:
-                clean_story_arcs[clean_story_arc] = number
-        cls._assign_or_pop(md, STORY_ARCS_METADATA_KEY, clean_story_arcs)
-
-    @classmethod
-    def _clean_identifiers(cls, md):
-        identifiers = md.get(IDENTIFIERS_KEY)
-        if not identifiers:
-            return
-        clean_identifiers = {}
-        for nid, identifier in identifiers.items():
-            clean_id_type = cls._clean_strfield(IdentifierType, "name", nid)
-            nss = identifier.get("nss")
-            clean_nss = cls._clean_strfield(Identifier, "nss", nss)
-            url = identifier.get("url")
-            clean_url = cls._clean_strfield(Identifier, "url", url)
-            if clean_nss:
-                clean_identifier = {"nss": nss}
-                if clean_url:
-                    clean_identifier["url"] = url
-                clean_identifiers[clean_id_type] = clean_identifier
-        cls._assign_or_pop(md, IDENTIFIERS_KEY, clean_identifiers)
-
-    @classmethod
-    def _clean_md(cls, md):
-        """Sanitize the metadata before import."""
-        cls._title_to_name(md)
-        md = cls._prune_extra_keys(md)
-        cls._clean_comic_positive_small_ints(md)
-        cls._clean_comic_small_ints(md)
-        cls._clean_comic_decimals(md)
-        cls._clean_strfields(md)
-        cls._clean_contributors(md)
-        cls._clean_story_arcs(md)
-        cls._clean_identifiers(md)
-        return md
-
-    def extract_and_clean(self, path, import_metadata):
-        """Extract metadata from comic and clean it for codex."""
-        md = {}
-        failed_import = {}
-        try:
-            if import_metadata:
-                with Comicbox(path) as cb:
-                    md = cb.to_dict()
-                    md = md.get("comicbox", {})
-                    if "file_type" not in md:
-                        md["file_type"] = cb.get_file_type()
-                    if "page_count" not in md:
-                        md["page_count"] = cb.get_page_count()
-            md["path"] = path
-            md = self._clean_md(md)
-        except (UnsupportedArchiveTypeError, BadRarFile, BadZipFile, OSError) as exc:
-            self.log.warning(f"Failed to import {path}: {exc}")
-            failed_import = {path: exc}
-        except Exception as exc:
-            self.log.exception(f"Failed to import: {path}")
-            failed_import = {path: exc}
-        if failed_import:
-            self.metadata[FIS].update(failed_import)
-        return md

codex/librarian/importer/failed_imports.py

@@ -1,165 +0,0 @@
-"""Update and create failed imports."""
-
-from pathlib import Path
-
-from django.db.models.functions import Now
-
-from codex.librarian.importer.const import FIS
-from codex.librarian.importer.deleted import DeletedImporter
-from codex.librarian.importer.status import ImportStatusTypes
-from codex.models import Comic, FailedImport
-from codex.status import Status
-
-_UPDATE_FIS = "update_fis"
-_CREATE_FIS = "create_fis"
-_DELETE_FI_PATHS = "delete_fi_paths"
-_BULK_UPDATE_FAILED_IMPORT_FIELDS = ("name", "stat", "updated_at")
-
-
-class FailedImportsImporter(DeletedImporter):
-    """Methods for failed imports."""
-
-    def _query_failed_import_deletes(self, existing_failed_import_paths):
-        """Calculate Deletes."""
-        untouched_failed_import_paths = existing_failed_import_paths - frozenset(
-            self.metadata[FIS].keys()
-        )
-
-        succeeded_failed_imports = frozenset(
-            Comic.objects.filter(
-                library=self.library, path__in=untouched_failed_import_paths
-            ).values_list("path", flat=True)
-        )
-
-        possibly_missing_failed_import_paths = (
-            untouched_failed_import_paths - succeeded_failed_imports
-        )
-        missing_failed_imports = set()
-        for path in possibly_missing_failed_import_paths:
-            if not Path(path).exists():
-                missing_failed_imports.add(path)
-
-        return succeeded_failed_imports | missing_failed_imports
-
-    def _query_failed_imports(
-        self,
-        fis,
-    ):
-        """Determine what to do with failed imports."""
-        if not self.metadata[FIS]:
-            return
-
-        # Remove the files deleted hack thing.
-        self.metadata[FIS].pop("files_deleted", None)
-
-        existing_failed_import_paths = set(
-            FailedImport.objects.filter(library=self.library).values_list(
-                "path", flat=True
-            )
-        )
-
-        # Calculate creates and updates
-        for path, exc in self.metadata[FIS].items():
-            if path in existing_failed_import_paths:
-                fis[_UPDATE_FIS][path] = exc
-            else:
-                fis[_CREATE_FIS][path] = exc
-
-        fis[_DELETE_FI_PATHS] |= self._query_failed_import_deletes(
-            existing_failed_import_paths
-        )
-        self.metadata.pop(FIS)
-
-        count = 0
-        for iterable in fis.values():
-            count += len(iterable)
-        return
-
-    def _bulk_update_failed_imports(self, update_failed_imports):
-        """Bulk update failed imports."""
-        if not update_failed_imports:
-            return
-        update_failed_import_objs = FailedImport.objects.filter(
-            library=self.library, path__in=update_failed_imports.keys()
-        ).only(*_BULK_UPDATE_FAILED_IMPORT_FIELDS)
-        if not update_failed_import_objs:
-            return
-        for fi in update_failed_import_objs:
-            try:
-                exc = update_failed_imports.pop(fi.path)
-                fi.set_reason(exc)
-                fi.presave()
-                fi.updated_at = Now()
-            except Exception as exc:
-                self.log.exception(
-                    f"Error preparing failed import update for {fi.path}"
-                )
-
-        FailedImport.objects.bulk_update(
-            update_failed_import_objs, fields=_BULK_UPDATE_FAILED_IMPORT_FIELDS
-        )
-        count = len(update_failed_import_objs)
-        if count:
-            self.log.info(f"Updated {count} old failed imports.")
-        return
-
-    def _bulk_create_failed_imports(self, create_failed_imports):
-        """Bulk create failed imports."""
-        if not create_failed_imports:
-            return 0
-        create_objs = []
-        for path, exc in create_failed_imports.items():
-            try:
-                fi = FailedImport(library=self.library, path=path, parent_folder=None)
-                fi.set_reason(exc)
-                fi.presave()
-                create_objs.append(fi)
-            except Exception as exc:
-                self.log.exception(f"Error preparing failed import create for {path}")
-        FailedImport.objects.bulk_create(
-            create_objs,
-            update_conflicts=True,
-            update_fields=_BULK_UPDATE_FAILED_IMPORT_FIELDS,
-            unique_fields=FailedImport._meta.unique_together[0],
-        )
-        count = len(create_objs)
-        if count:
-            self.log.info(f"Added {count} comics to failed imports.")
-        return count
-
-    def _bulk_cleanup_failed_imports(self, delete_failed_imports_paths):
-        """Remove FailedImport objects that have since succeeded."""
-        if not delete_failed_imports_paths:
-            return 0
-        # Cleanup FailedImports that were actually successful
-        qs = FailedImport.objects.filter(library=self.library).filter(
-            path__in=delete_failed_imports_paths
-        )
-        qs.delete()
-        count = len(delete_failed_imports_paths)
-        if count:
-            self.log.info(f"Cleaned up {count} failed imports from {self.library.path}")
-        return count
-
-    def fail_imports(self):
-        """Handle failed imports."""
-        fis = self.metadata.get(FIS)
-        if not fis:
-            return False
-        status = Status(ImportStatusTypes.FAILED_IMPORTS)
-        self.status_controller.start(status)
-        created_count = 0
-
-        try:
-            fis = {_UPDATE_FIS: {}, _CREATE_FIS: {}, _DELETE_FI_PATHS: set()}
-            if self.task.files_deleted:
-                # if any files were deleted. Run the failed import check
-                self.metadata[FIS]["files_deleted"] = True
-            self._query_failed_imports(fis)
-            self._bulk_update_failed_imports(fis[_UPDATE_FIS])
-            self._bulk_create_failed_imports(fis[_CREATE_FIS])
-            self._bulk_cleanup_failed_imports(fis[_DELETE_FI_PATHS])
-        except Exception:
-            self.log.exception("Processing failed imports")
-        self.status_controller.finish(status)
-        return bool(created_count)

codex/librarian/importer/importer.py

@@ -1,101 +0,0 @@
-"""The main importer class."""
-
-from time import time
-
-from django.core.cache import cache
-from humanize import naturaldelta
-
-from codex.librarian.importer.moved import MovedImporter
-from codex.librarian.importer.status import ImportStatusTypes
-from codex.librarian.notifier.tasks import (
-    FAILED_IMPORTS_CHANGED_TASK,
-    LIBRARY_CHANGED_TASK,
-)
-from codex.librarian.search.tasks import SearchIndexUpdateTask
-from codex.librarian.tasks import DelayedTasks
-
-
-class ComicImporter(MovedImporter):
-    """Initialize, run and finish a bulk import."""
-
-    ##########
-    # FINISH #
-    ##########
-    def _finish_apply_status(self):
-        """Finish all librarian statuses."""
-        self.library.update_in_progress = False
-        self.library.save()
-        self.status_controller.finish_many(ImportStatusTypes.values)
-
-    def _finish_apply(self, new_failed_imports, imported_count):
-        """Perform final tasks when the apply is done."""
-        if self.changed:
-            cache.clear()
-            elapsed_time = time() - self.start_time.timestamp()
-            elapsed = naturaldelta(elapsed_time)
-            log_txt = f"Updated library {self.library.path} in {elapsed}."
-            if imported_count:
-                cps = round(imported_count / elapsed_time, 1)
-                log_txt += (
-                    f" Imported {imported_count} comics at {cps} comics per second."
-                )
-            else:
-                log_txt += " No comics to import."
-            self.log.info(log_txt)
-            self.librarian_queue.put(LIBRARY_CHANGED_TASK)
-
-            # Wait to start the search index update in case more updates are incoming.
-            until = time() + 1
-            delayed_search_task = DelayedTasks(
-                until, (SearchIndexUpdateTask(rebuild=False),)
-            )
-            self.librarian_queue.put(delayed_search_task)
-        else:
-            self.log.info("No updates neccissary.")
-        if new_failed_imports:
-            self.librarian_queue.put(FAILED_IMPORTS_CHANGED_TASK)
-
-    def apply(self):
-        """Bulk import comics."""
-        try:
-            self.init_apply()
-            self.move_and_modify_dirs()
-
-            #############
-            # AGGREGATE #
-            #############
-            self.get_aggregate_metadata()
-
-            #########
-            # QUERY #
-            #########
-            self.query_all_missing_fks()
-            self.query_missing_custom_covers()
-
-            #####################
-            # UPDATE AND CREATE #
-            #####################
-            self.create_all_fks()
-            self.update_custom_covers()
-            self.create_custom_covers()
-
-            imported_count = self.bulk_update_comics()
-            imported_count += self.bulk_create_comics()
-
-            ########
-            # LINK #
-            ########
-            self.bulk_query_and_link_comic_m2m_fields()
-            self.link_custom_covers()
-            self.changed += imported_count
-
-            new_failed_imports = self.fail_imports()
-
-            deleted_comic_groups = self.delete()
-
-            self.update_all_groups(deleted_comic_groups, self.start_time)
-
-        finally:
-            self._finish_apply_status()
-
-        self._finish_apply(new_failed_imports, imported_count)

codex/librarian/importer/importerd.py

@@ -1,122 +0,0 @@
-"""Bulk import and move comics and folders."""
-
-from pathlib import Path
-
-from django.utils import timezone
-
-from codex.librarian.importer.importer import ComicImporter
-from codex.librarian.importer.status import ImportStatusTypes
-from codex.librarian.importer.tasks import (
-    AdoptOrphanFoldersTask,
-    ImportDBDiffTask,
-    LazyImportComicsTask,
-    UpdateGroupsTask,
-)
-from codex.librarian.janitor.tasks import JanitorAdoptOrphanFoldersFinishedTask
-from codex.librarian.notifier.tasks import LIBRARY_CHANGED_TASK
-from codex.models import Comic, Folder, Library
-from codex.status import Status
-from codex.threads import QueuedThread
-
-
-class ComicImporterThread(QueuedThread):
-    """A worker to handle all bulk database updates."""
-
-    def _create_importer(self, task):
-        return ComicImporter(task, self.log_queue, self.librarian_queue)
-
-    def _import(self, task):
-        """Run an import task."""
-        importer = self._create_importer(task)
-        importer.apply()
-
-    def _lazy_import_metadata(self, task):
-        """Kick off an import task for just these books."""
-        import_comics = Comic.objects.filter(pk__in=task.pks).only("path", "library_id")
-        library_path_map = {}
-        for import_comic in import_comics:
-            library_id = import_comic.library_id  # type: ignore[reportAttributeAccessIssue]
-            if library_id not in library_path_map:
-                library_path_map[library_id] = set()
-            library_path_map[library_id].add(import_comic.path)
-
-        for library_id, paths in library_path_map.items():
-            # An abridged import task.
-            task = ImportDBDiffTask(
-                library_id=library_id,
-                files_modified=frozenset(paths),
-                force_import_metadata=True,
-            )
-            self._import(task)
-
-    def _update_groups(self, task):
-        pks = Library.objects.filter(covers_only=False).values_list("pk", flat=True)
-        start_time = task.start_time if task.start_time else timezone.now()
-        for pk in pks:
-            task = ImportDBDiffTask(library_id=pk)
-            importer = self._create_importer(task)
-            importer.update_all_groups({}, start_time)
-        self.librarian_queue.put(LIBRARY_CHANGED_TASK)
-
-    def _adopt_orphan_folders_for_library(self, library):
-        """Adopt orphan folders for one library."""
-        orphan_folder_paths = (
-            Folder.objects.filter(library=library, parent_folder=None)
-            .exclude(path=library.path)
-            .values_list("path", flat=True)
-        )
-        # Move in place
-        # Exclude deleted folders
-        folders_moved = {
-            path: path for path in orphan_folder_paths if Path(path).is_dir()
-        }
-
-        if folders_moved:
-            self.log.debug(
-                f"{len(folders_moved)} orphan folders found in {library.path}"
-            )
-        else:
-            self.log.debug(f"No orphan folders in {library.path}")
-            return False
-
-        # An abridged import task.
-        task = ImportDBDiffTask(
-            library_id=library.pk,
-            dirs_moved=folders_moved,
-        )
-        importer = self._create_importer(task)
-        # Only run the moved task.
-        importer.bulk_folders_moved()
-        return True
-
-    def _adopt_orphan_folders(self, janitor: bool):
-        """Find orphan folders and move them into their correct place."""
-        status = Status(ImportStatusTypes.ADOPT_FOLDERS)
-        moved_status = Status(ImportStatusTypes.DIRS_MOVED)
-        try:
-            self.status_controller.start_many((status, moved_status))
-            libraries = Library.objects.filter(covers_only=False).only("path")
-            for library in libraries.iterator():
-                folders_left = True
-                while folders_left:
-                    # Run until there are no orphan folders
-                    folders_left = self._adopt_orphan_folders_for_library(library)
-        finally:
-            self.status_controller.finish_many((moved_status, status))
-            if janitor:
-                next_task = JanitorAdoptOrphanFoldersFinishedTask()
-                self.librarian_queue.put(next_task)
-
-    def process_item(self, item):
-        """Run the updater."""
-        task = item
-        if isinstance(task, ImportDBDiffTask):
-            self._import(task)
-        elif isinstance(task, LazyImportComicsTask):
-            self._lazy_import_metadata(task)
-        elif isinstance(task, UpdateGroupsTask):
-            self._update_groups(task)
-        elif isinstance(task, AdoptOrphanFoldersTask):
-            self._adopt_orphan_folders(janitor=task.janitor)
-        else:
-            self.log.warning(f"Bad task sent to library updater {task}")