codex 1.7.14__py3-none-any.whl → 1.8.0a2__py3-none-any.whl

codex/librarian/importer/tasks.py

@@ -1,55 +0,0 @@
-"""DB Import Tasks."""
-
-from collections.abc import Mapping
-from dataclasses import dataclass, field
-from datetime import datetime
-
-
-@dataclass
-class ImportTask:
-    """Tasks for the updater."""
-
-
-@dataclass
-class ImportDBDiffTask(ImportTask):
-    """For sending to the updater."""
-
-    library_id: int
-
-    dirs_moved: Mapping[str, str] = field(default_factory=dict)
-    dirs_modified: frozenset[str] = frozenset()
-    # dirs_created: frozenset[str] | None = frozenset() # noqa: ERA001
-    dirs_deleted: frozenset[str] = frozenset()
-
-    files_moved: Mapping[str, str] = field(default_factory=dict)
-    files_modified: frozenset[str] = frozenset()
-    files_created: frozenset[str] = frozenset()
-    files_deleted: frozenset[str] = frozenset()
-
-    covers_moved: Mapping[str, str] = field(default_factory=dict)
-    covers_modified: frozenset[str] = frozenset()
-    covers_created: frozenset[str] = frozenset()
-    covers_deleted: frozenset[str] = frozenset()
-
-    force_import_metadata: bool = False
-
-
-@dataclass
-class LazyImportComicsTask(ImportTask):
-    """Lazy import of metadaa for existing comics."""
-
-    pks: frozenset[int]
-
-
-@dataclass
-class AdoptOrphanFoldersTask(ImportTask):
-    """Move orphaned folders into a correct tree position."""
-
-    janitor: bool = False
-
-
-@dataclass
-class UpdateGroupsTask(ImportTask):
-    """Force the update of group timestamp."""
-
-    start_time: datetime | None = None

codex/librarian/janitor/__init__.py

@@ -1 +0,0 @@
-"""Maintenance tasks."""

codex/librarian/janitor/cleanup.py

@@ -1,167 +0,0 @@
-"""Clean up the database after moves or imports."""
-
-import logging
-from pathlib import Path
-from time import time
-from types import MappingProxyType
-
-from django.contrib.sessions.models import Session
-from django.db.models.functions.datetime import Now
-
-from codex.librarian.janitor.status import JanitorStatusTypes
-from codex.models import (
-    AgeRating,
-    Character,
-    Contributor,
-    ContributorPerson,
-    ContributorRole,
-    Country,
-    Folder,
-    Genre,
-    Identifier,
-    IdentifierType,
-    Imprint,
-    Language,
-    Location,
-    OriginalFormat,
-    Publisher,
-    ScanInfo,
-    Series,
-    SeriesGroup,
-    StoryArc,
-    StoryArcNumber,
-    Tag,
-    Tagger,
-    Team,
-    Volume,
-)
-from codex.models.bookmark import Bookmark
-from codex.models.paths import CustomCover
-from codex.status import Status
-from codex.worker_base import WorkerBaseMixin
-
-_COMIC_FK_CLASSES = (
-    AgeRating,
-    Country,
-    Contributor,
-    Character,
-    Genre,
-    Folder,
-    Language,
-    Location,
-    Identifier,
-    Imprint,
-    OriginalFormat,
-    Publisher,
-    Series,
-    SeriesGroup,
-    ScanInfo,
-    StoryArcNumber,
-    Tagger,
-    Tag,
-    Team,
-    Volume,
-)
-_CONTRIBUTOR_FK_CLASSES = (ContributorRole, ContributorPerson)
-_STORY_ARC_NUMBER_FK_CLASSES = (StoryArc,)
-_IDENTIFIER_FK_CLASSES = (IdentifierType,)
-TOTAL_NUM_FK_CLASSES = (
-    len(_COMIC_FK_CLASSES)
-    + len(_CONTRIBUTOR_FK_CLASSES)
-    + len(_STORY_ARC_NUMBER_FK_CLASSES)
-    + len(_IDENTIFIER_FK_CLASSES)
-)
-
-CLEANUP_MAP = MappingProxyType(
-    {
-        "comic": _COMIC_FK_CLASSES,
-        "contributor": _CONTRIBUTOR_FK_CLASSES,
-        "storyarcnumber": _STORY_ARC_NUMBER_FK_CLASSES,
-        "identifier": _IDENTIFIER_FK_CLASSES,
-    }
-)
-
-
-class CleanupMixin(WorkerBaseMixin):
-    """Cleanup methods for Janitor."""
-
-    def _bulk_cleanup_fks(self, models, field_name, status):
-        """Remove foreign keys that aren't used anymore."""
-        for model in models:
-            filter_dict = {f"{field_name}__isnull": True}
-            qs = model.objects.filter(**filter_dict)
-            count = qs.count()
-            qs.delete()
-            if count:
-                self.log.info(f"Deleted {count} orphan {model.__name__}s")
-            status.complete += count
-            self.status_controller.update(status)
-
-    def cleanup_fks(self):
-        """Clean up unused foreign keys."""
-        status = Status(JanitorStatusTypes.CLEANUP_FK, 0, TOTAL_NUM_FK_CLASSES)
-        try:
-            self.status_controller.start(status)
-            self.log.debug("Cleaning up unused foreign keys...")
-            for field_name, models in CLEANUP_MAP.items():
-                self._bulk_cleanup_fks(models, field_name, status)
-            level = logging.INFO if status.complete else logging.DEBUG
-            self.log.log(level, f"Cleaned up {status.complete} unused foreign keys.")
-        finally:
-            self.status_controller.finish(status)
-
-    def cleanup_custom_covers(self):
-        """Clean up unused custom covers."""
-        covers = CustomCover.objects.only("path")
-        status = Status(JanitorStatusTypes.CLEANUP_COVERS, 0, covers.count())
-        delete_pks = []
-        try:
-            self.status_controller.start(status)
-            self.log.debug("Cleaning up db custom covers with no source images...")
-            for cover in covers.iterator():
-                if not Path(cover.path).exists():
-                    delete_pks.append(cover.pk)
-                status.increment_complete()
-            delete_qs = CustomCover.objects.filter(pk__in=delete_pks)
-            count = delete_qs.count()
-            delete_qs.delete()
-            level = logging.INFO if status.complete else logging.DEBUG
-            self.log.log(level, f"Deleted {count} CustomCovers without source images.")
-        finally:
-            until = time() + 1
-            self.status_controller.finish(status, until=until)
-
-    def cleanup_sessions(self):
-        """Delete corrupt sessions."""
-        status = Status(JanitorStatusTypes.CLEANUP_SESSIONS)
-        try:
-            self.status_controller.start(status)
-            count, _ = Session.objects.filter(expire_date__lt=Now()).delete()
-            if count:
-                self.log.info(f"Deleted {count} expired sessions.")
-
-            bad_session_keys = set()
-            for encoded_session in Session.objects.all():
-                session = encoded_session.get_decoded()
-                if not session:
-                    bad_session_keys.add(encoded_session.session_key)
-
-            if bad_session_keys:
-                bad_sessions = Session.objects.filter(session_key__in=bad_session_keys)
-                count, _ = bad_sessions.delete()
-                self.log.info(f"Deleted {count} corrupt sessions.")
-        finally:
-            until = time() + 1
-            self.status_controller.finish(status, until=until)
-
-    def cleanup_orphan_bookmarks(self):
-        """Delete bookmarks without users or sessions."""
-        status = Status(JanitorStatusTypes.CLEANUP_BOOKMARKS)
-        try:
-            self.status_controller.start(status)
-            orphan_bms = Bookmark.objects.filter(session=None, user=None)
-            if count := orphan_bms.count():
-                orphan_bms.delete()
-                self.log.info(f"Deleted {count} orphan bookmarks.")
-        finally:
-            self.status_controller.finish(status)

codex/librarian/janitor/failed_imports.py

@@ -1,27 +0,0 @@
-"""Force update events for failed imports."""
-
-from watchdog.events import FileModifiedEvent
-
-from codex.librarian.watchdog.tasks import WatchdogEventTask
-from codex.models import FailedImport, Library
-from codex.worker_base import WorkerBaseMixin
-
-
-class UpdateFailedImportsMixin(WorkerBaseMixin):
-    """Methods for updating failed imports."""
-
-    def _force_update_failed_imports(self, library_id):
-        """Force update events for failed imports in a library."""
-        failed_import_paths = FailedImport.objects.filter(
-            library=library_id
-        ).values_list("path", flat=True)
-        for path in failed_import_paths:
-            event = FileModifiedEvent(path)
-            task = WatchdogEventTask(library_id, event)
-            self.librarian_queue.put(task)
-
-    def force_update_all_failed_imports(self):
-        """Force update events for failed imports in every library."""
-        pks = Library.objects.filter(covers_only=False).values_list("pk", flat=True)
-        for pk in pks:
-            self._force_update_failed_imports(pk)

codex/librarian/janitor/integrity.py

@@ -1,378 +0,0 @@
-"""Database integrity checks and remedies."""
-# Uses app.get_model() because functions may also be called before the models are ready on startup.
-
-import re
-from typing import TYPE_CHECKING
-
-from django.apps import apps
-from django.db import DEFAULT_DB_ALIAS, connections
-from django.db.models.functions import Now
-from django.db.utils import OperationalError
-
-from codex.librarian.janitor.status import JanitorStatusTypes
-from codex.librarian.janitor.tasks import JanitorFTSRebuildTask
-from codex.logger.logger import get_logger
-from codex.settings.settings import (
-    CONFIG_PATH,
-    CUSTOM_COVERS_DIR,
-    DB_PATH,
-)
-from codex.status import Status
-from codex.worker_base import WorkerBaseMixin
-
-if TYPE_CHECKING:
-    from django.db.models.manager import BaseManager
-
-    from codex.models.base import BaseModel
-    from codex.models.comic import Comic
-
-
-REPAIR_FLAG_PATH = CONFIG_PATH / "rebuild_db"
-REBUILT_DB_PATH = DB_PATH.parent / (DB_PATH.name + ".rebuilt")
-BACKUP_DB_PATH = DB_PATH.parent / (DB_PATH.name + ".bak")
-DUMP_LINE_MATCHER = re.compile("TRANSACTION|ROLLBACK|COMMIT")
-
-LOG = get_logger(__name__)
-
-_FTS_INSERT_TMPL = "INSERT INTO codex_comicfts (codex_comicfts) VALUES('%s');"
-_PRAGMA_TMPL = "PRAGMA %s;"
-
-
-def _exec_sql(sql):
-    """Run sql on an potentially unready database.."""
-    connection = connections[DEFAULT_DB_ALIAS]
-    connection.prepare_database()
-    with connection.cursor() as cursor:
-        cursor.execute("PRAGMA wal_checkpoint(TRUNCATE)")
-        cursor.execute(sql)
-        return cursor.fetchall()
-
-
-def _foreign_key_check(log):
-    """Get table and row ids from foreign_key_check."""
-    sql = _PRAGMA_TMPL % "foreign_key_check"
-    if results := _exec_sql(sql):
-        log.warning(f"Found {len(results)} database rows with illegal foreign keys.")
-        log.debug(results)
-    else:
-        log.info("Database passed foreign key check.")
-    return results
-
-
-def _compile_foreign_key_results(results, log):
-    bad_fk_rels = {}
-    bad_m2m_rows = {}
-
-    for row in results:
-        table_name, rowid, _parent, fkid = row
-        if fkid:
-            if table_name not in bad_fk_rels:
-                bad_fk_rels[table_name] = {}
-            if rowid not in bad_fk_rels[table_name]:
-                bad_fk_rels[table_name][rowid] = set()
-            bad_fk_rels[table_name][rowid].add(fkid)
-        else:
-            if table_name not in bad_m2m_rows:
-                bad_m2m_rows[table_name] = set()
-            bad_m2m_rows[table_name].add(rowid)
-
-    if bad_fk_rels:
-        log.debug(f"Found {len(bad_fk_rels)} tables with illegal foreign keys.")
-    if bad_m2m_rows:
-        log.debug(
-            f"Found {len(bad_m2m_rows)} many to many through tables with illegal relations."
-        )
-
-    return bad_fk_rels, bad_m2m_rows
-
-
-def _get_model(table_name):
-    """Get a django model from the table name."""
-    app_name, model_name = table_name.split("_", 1)
-    app_config = apps.get_app_config(app_name)
-    try:
-        from_rel_model_name, to_rel_model_name = model_name.split("_", 1)
-    except ValueError:
-        from_rel_model_name = to_rel_model_name = ""
-    if from_rel_model_name in ("comic", "library"):
-        # Use the from model to create an m2m through model because I don't know a simpler way.
-        from_model = app_config.get_model(from_rel_model_name)
-        field_name = to_rel_model_name.replace("_", "")
-        model = getattr(from_model, field_name).through
-    else:
-        # Not an m2m through model.
-        model_name = model_name.replace("_", "")
-        model = app_config.get_model(model_name)
-    return model
-
-
-def _null_bad_fk_rels_table(table_name, bad_rows, log):
-    """Null bad foreign key relations by table."""
-    fix_comic_pks = set()
-    model = _get_model(table_name)
-    fields = model._meta.fields
-    field_names = {}
-    update_objs = []
-    objs: BaseManager[BaseModel] = model.objects.filter(pk__in=bad_rows.keys())  # type: ignore[reportAssignmentType]
-    update_fields = {"updated_at"}
-    now = Now()
-    for obj in objs:
-        try:
-            fkids = bad_rows.get(obj.pk)
-            if not fkids:
-                continue
-            for fkid in fkids:
-                if fkid not in field_names:
-                    field_names[fkid] = fields[fkid].name
-                field_name = field_names[fkid]
-                setattr(obj, field_name, None)
-                update_fields.add(field_name)
-            obj.updated_at = now
-            update_objs.append(obj)
-        except Exception as exc:
-            log.warning(
-                f"Unable to null bad foreign keys in {table_name}:{obj.pk} - {exc}"
-            )
-        if table_name == "comic":
-            fix_comic_pks.add(obj.pk)
-
-    if update_objs:
-        model.objects.bulk_update(update_objs, sorted(update_fields))
-        log.info(
-            f"Removed illegal foreign key relations from {len(update_objs)} {table_name}s"
-        )
-    return fix_comic_pks
-
-
-def _null_bad_fk_rels(bad_fk_rels, log):
-    """Null bad foreign key relations."""
-    fix_comic_pks = set()
-    for table_name, bad_rows in bad_fk_rels.items():
-        try:
-            fix_comic_pks |= _null_bad_fk_rels_table(table_name, bad_rows, log)
-        except Exception:
-            pks = sorted(bad_rows.keys())
-            log.exception(
-                f"Unable to null {len(pks)} {table_name} rows with bad foreign keys"
-            )
-            log.error(f"{table_name}: {pks}")  # noqa: TRY400
-    return fix_comic_pks
-
-
-def _delete_bad_m2m_rows_table(table_name, ids, fix_comic_pks, log):
-    """Delete illegal foreign keys for one table."""
-    model = _get_model(table_name)
-
-    count = len(ids)
-    qs = model.objects.filter(id__in=ids)
-    try:
-        comic_ids = qs.values_list("comic_id", flat=True)
-        fix_comic_pks |= set(comic_ids)
-    except (OperationalError, AttributeError):
-        log.exception("Add comic ids to fix list.")
-
-    qs.delete()
-    log.info(f"Deleted {count} {table_name} rows that failed integrity check.")
-    return count
-
-
-def _delete_bad_m2m_rows(bad_m2m_rows, log):
-    """Delete illegal foreign keys for all tables."""
-    fix_comic_pks = set()
-    count = 0
-    for table_name, ids in bad_m2m_rows.items():
-        try:
-            count += _delete_bad_m2m_rows_table(table_name, ids, fix_comic_pks, log)
-        except Exception:
-            log.exception(f"Could not delete bad foreign keys in table {table_name}")
-    if count:
-        log.info(f"Removed {count} bad foreign key relations.")
-    return fix_comic_pks
-
-
-def _mark_comics_for_update(fix_comic_pks, log):
-    """Mark comics with altered foreign keys for update."""
-    if not fix_comic_pks:
-        return
-    comic_model = apps.get_model(app_label="codex", model_name="comic")
-    outdated_comics: BaseManager[Comic] = comic_model.objects.filter(
-        pk__in=fix_comic_pks
-    ).only(  # type: ignore[reportAssignmentType]
-        "stat", "updated_at"
-    )
-    if not outdated_comics:
-        return
-
-    update_comics = []
-    now = Now()
-    for comic in outdated_comics:
-        stat_list = comic.stat
-        if not stat_list:
-            continue
-        stat_list[8] = 0.0
-        comic.stat = stat_list  # type: ignore[reportAttributeAccessIssue]
-        comic.updated_at = now
-        update_comics.append(comic)
-
-    if update_comics:
-        count = comic_model.objects.bulk_update(
-            update_comics, fields=["stat", "updated_at"]
-        )
-        log.info(f"Marked {count} comics with bad relations for update by poller.")
-
-
-def fix_foreign_keys(log=None):
-    """Foreign Key Check."""
-    if not log:
-        log = LOG
-    try:
-        results = _foreign_key_check(log)
-        bad_fk_rels, bad_m2m_rows = _compile_foreign_key_results(results, log)
-    except Exception:
-        log.exception("Integrity: foreign_key_check")
-        return
-    fix_comic_pks = _null_bad_fk_rels(bad_fk_rels, log)
-    fix_comic_pks |= _delete_bad_m2m_rows(bad_m2m_rows, log)
-    try:
-        _mark_comics_for_update(fix_comic_pks, log)
-    except Exception:
-        LOG.exception("Could not mark comics with bad relations for update")
-
-
-def _repair_extra_custom_cover_libraries(library_model, log):
-    """Attempt to remove the bad ones, probably futile."""
-    delete_libs = library_model.objects.filter(covers_only=True).exclude(
-        path=CUSTOM_COVERS_DIR
-    )
-    count = delete_libs.count()
-    if count:
-        delete_libs.delete()
-        log.warning(
-            f"Removed {count} duplicate custom cover libraries pointing to unused custom cover dirs."
-        )
-
-
-def cleanup_custom_cover_libraries(log=None):
-    """Cleanup extra custom cover libraries."""
-    if not log:
-        log = LOG
-    try:
-        try:
-            library_model = apps.get_model("codex", "library")
-        except LookupError:
-            log.debug("Library model doesn't exist yet.")
-            return
-        if not library_model or not hasattr(library_model, "covers_only"):
-            log.debug("Library model doesn't support custom cover library yet.")
-            return
-        _repair_extra_custom_cover_libraries(library_model, log)
-
-        custom_cover_libraries = library_model.objects.filter(covers_only=True)
-        count = custom_cover_libraries.count()
-        if count <= 1:
-            return
-        custom_cover_libraries.delete()
-        log.warning(
-            f"Removed all ({count}) custom cover libraries, Unable to determine valid one. Will recreate upon startup."
-        )
-    except Exception as exc:
-        log.warning(f"Failed to check custom cover library for integrity - {exc}")
-
-
-def _is_integrity_ok(results):
-    return (
-        results and len(results) == 1 and len(results[0]) == 1 and results[0][0] == "ok"
-    )
-
-
-def integrity_check(long, log=None):
-    """Run sqlite3 integrity check."""
-    if not log:
-        log = LOG
-    pragma = "integrity_check" if long else "quick_check"
-    sql = _PRAGMA_TMPL % pragma
-    log.debug(f"Running database '{sql}'...")
-    results = _exec_sql(sql)
-
-    if _is_integrity_ok(results):
-        length = "" if long else "quick "
-        log.info(f"Database passed {length}integrity check.")
-    else:
-        log.warning(f"Database '{sql}' returned results:")
-        log.warning(results)
-        log.warning(
-            "See the README for database rebuild instructions if the above warning looks severe."
-        )
-
-
-def fts_rebuild(log=None):
-    """FTS Rebuild."""
-    sql = _FTS_INSERT_TMPL % "rebuild"
-    if not log:
-        log = LOG
-    _exec_sql(sql)
-    log.info("Rebuilt FTS Virtual Table.")
-
-
-def fts_integrity_check(log=None):
-    """Run sqlite3 fts integrity check."""
-    if not log:
-        log = LOG
-    results = []
-    sql = _FTS_INSERT_TMPL % "integrity-check"
-    success = False
-    try:
-        results = _exec_sql(sql)
-        if results:
-            # I'm not sure if this raises or puts the error in the results.
-            raise ValueError(results)  # noqa: TRY301
-    except Exception:
-        log.exception("Full Text Search Index failed integrity check.")
-    else:
-        log.info("Full Text Search Index passed integrity check.")
-        success = True
-    return success
-
-
-class IntegrityMixin(WorkerBaseMixin):
-    """Integrity Check Mixin."""
-
-    def foreign_key_check(self):
-        """Foreign Key Check task."""
-        status = Status(JanitorStatusTypes.INTEGRITY_FK)
-        try:
-            self.status_controller.start(status)
-            fix_foreign_keys(self.log)
-
-        finally:
-            self.status_controller.finish(status)
-
-    def integrity_check(self, long: bool):
-        """Integrity check task."""
-        subtitle = "" if long else "Quick"
-        status = Status(JanitorStatusTypes.INTEGRITY_CHECK, subtitle=subtitle)
-        try:
-            self.status_controller.start(status)
-            integrity_check(long, self.log)
-        finally:
-            self.status_controller.finish(status)
-
-    def fts_rebuild(self):
-        """FTS rebuild task."""
-        status = Status(JanitorStatusTypes.FTS_REBUILD)
-        try:
-            self.status_controller.start(status)
-            fts_rebuild(self.log)
-        finally:
-            self.status_controller.finish(status)
-
-    def fts_integrity_check(self):
-        """FTS integrity check task."""
-        status = Status(JanitorStatusTypes.FTS_INTEGRITY_CHECK)
-        try:
-            self.status_controller.start(status)
-            success = fts_integrity_check(self.log)
-            if not success:
-                self.librarian_queue.put(JanitorFTSRebuildTask())
-        finally:
-            self.status_controller.finish(status)