codex 1.7.15__py3-none-any.whl → 1.8.0a2__py3-none-any.whl

codex-1.8.0a2.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.27.0
+Root-Is-Purelib: true
+Tag: py3-none-any

codex-1.8.0a2.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+codex = codex.run:main

codex/db.py

@@ -1,134 +0,0 @@
-"""Low level database utilities."""
-
-import sqlite3
-import subprocess
-
-from django.core.management import call_command
-from django.db import DEFAULT_DB_ALIAS, connection, connections
-from django.db.migrations.executor import MigrationExecutor
-
-from codex.librarian.janitor.integrity import (
-    cleanup_custom_cover_libraries,
-    fix_foreign_keys,
-    fts_integrity_check,
-    fts_rebuild,
-    integrity_check,
-)
-from codex.librarian.janitor.janitor import Janitor
-from codex.librarian.mp_queue import LIBRARIAN_QUEUE
-from codex.logger.logger import get_logger
-from codex.logger.mp_queue import LOG_QUEUE
-from codex.settings.settings import (
-    BACKUP_DB_PATH,
-    CONFIG_PATH,
-    DB_PATH,
-    FIX_FOREIGN_KEYS,
-    FTS_INTEGRITY_CHECK,
-    FTS_REBUILD,
-    INTEGRITY_CHECK,
-)
-from codex.version import VERSION
-
-_REPAIR_FLAG_PATH = CONFIG_PATH / "rebuild_db"
-_REBUILT_DB_PATH = DB_PATH.parent / (DB_PATH.name + ".rebuilt")
-_REPAIR_ARGS = ("sqlite3", DB_PATH, ".recover")
-
-LOG = get_logger(__name__)
-
-
-def _has_unapplied_migrations():
-    """Check if any migrations are outstanding."""
-    try:
-        connection = connections[DEFAULT_DB_ALIAS]
-        connection.prepare_database()
-        executor = MigrationExecutor(connection)
-        targets = [
-            key for key in executor.loader.graph.leaf_nodes() if key[0] == "codex"
-        ]
-        plan = executor.migration_plan(targets)
-    except Exception as exc:
-        LOG.warning(f"has_unapplied_migrations(): {exc}")
-        return False
-    else:
-        return bool(plan)
-
-
-def _get_backup_db_path(prefix):
-    suffix = f".{prefix}{BACKUP_DB_PATH.suffix}"
-    return BACKUP_DB_PATH.with_suffix(suffix)
-
-
-def _backup_db_before_migration():
-    """If there are migrations to do, backup the db."""
-    backup_path = _get_backup_db_path(f"before-v{VERSION}")
-    janitor = Janitor(LOG_QUEUE, LIBRARIAN_QUEUE)
-    janitor.backup_db(show_status=False, backup_path=backup_path)
-    LOG.info("Backed up database before migrations")
-
-
-def _repair_db():
-    """Run integrity checks on startup."""
-    if FIX_FOREIGN_KEYS:
-        fix_foreign_keys()
-    if INTEGRITY_CHECK:
-        integrity_check(long=True)
-    success = fts_integrity_check() if FTS_INTEGRITY_CHECK else True
-    if FTS_REBUILD or not success:
-        fts_rebuild()
-    cleanup_custom_cover_libraries()
-
-
-def _rebuild_db():
-    """Dump and rebuild the database."""
-    # Drastic
-    if not _REPAIR_FLAG_PATH.exists():
-        return False
-
-    LOG.warning("REBUILDING DATABASE!!")
-    _REBUILT_DB_PATH.unlink(missing_ok=True)
-    recover_proc = subprocess.Popen(_REPAIR_ARGS, stdout=subprocess.PIPE)  # noqa: S603
-    with sqlite3.connect(_REBUILT_DB_PATH) as new_db_conn:
-        new_db_cur = new_db_conn.cursor()
-        if recover_proc.stdout:
-            for line in recover_proc.stdout:
-                row = line.decode().strip()
-                replaced_row = (
-                    "PRAGMA writable_schema = reset;"
-                    if row == "PRAGMA writable_schema = off;"
-                    else row
-                )
-                new_db_cur.execute(replaced_row)
-        new_db_cur.close()
-    if recover_proc.stdout:
-        recover_proc.stdout.close()
-        recover_proc.wait(timeout=15)
-
-    backup_path = _get_backup_db_path("before-rebuild")
-    DB_PATH.rename(backup_path)
-    LOG.info("Backed up old db to %s", backup_path)
-    _REBUILT_DB_PATH.replace(DB_PATH)
-    _REPAIR_FLAG_PATH.unlink(missing_ok=True)
-    LOG.info("Rebuilt database.")
-    LOG.info("You may start codex normally now.")
-    for handler in LOG.handlers:
-        handler.flush()
-    return True
-
-
-def ensure_db_schema():
-    """Ensure the db is good and up to date."""
-    LOG.info("Ensuring database is correct and up to date...")
-    table_names = connection.introspection.table_names()
-    if db_exists := "django_migrations" in table_names:
-        if _rebuild_db():
-            return False
-        _repair_db()
-
-    if not db_exists or _has_unapplied_migrations():
-        if db_exists:
-            _backup_db_before_migration()
-        call_command("migrate")
-    else:
-        LOG.info("Database up to date.")
-    LOG.info("Database ready.")
-    return True

codex/exceptions.py

@@ -1,88 +0,0 @@
-"""Special Redirect Error."""
-
-from collections.abc import Mapping
-from copy import deepcopy
-from pprint import pformat
-
-from django.core.validators import EMPTY_VALUES
-from django.shortcuts import redirect
-from django.urls import reverse
-from django.utils.http import urlencode
-from rest_framework.exceptions import APIException
-from rest_framework.status import HTTP_303_SEE_OTHER
-
-from codex.choices.browser import DEFAULT_BROWSER_ROUTE
-from codex.logger.logger import get_logger
-from codex.serializers.fields.browser import BreadcrumbsField
-from codex.serializers.route import RouteSerializer
-from codex.views.util import pop_name
-
-LOG = get_logger(__name__)
-_OPDS_REDIRECT_SETTINGS_KEYS = ("order_by", "top_group")
-_REDIRECT_SETTINGS_KEYS = ("breadcrumbs", *_OPDS_REDIRECT_SETTINGS_KEYS)
-
-
-class SeeOtherRedirectError(APIException):
-    """Redirect for 303."""
-
-    status_code = HTTP_303_SEE_OTHER
-    default_code = "redirect"
-    default_detail = "redirect to a valid route"
-
-    def __init__(self, detail):
-        """Create a response to pass to the exception handler."""
-        # Copy to edit and not write over refs
-        detail = dict(detail)
-
-        # Get route params
-        route = detail.get("route", {})
-        params = route.get("params", DEFAULT_BROWSER_ROUTE)
-        params = pop_name(params)
-        route = deepcopy(route)
-        route["params"] = params
-
-        # For OPDS
-        self.route_kwargs = params
-
-        serializer = RouteSerializer(params)
-        route["params"] = serializer.data
-        detail["route"] = route
-
-        filtered_settings = {}
-        settings = detail.get("settings", {})
-        for key in _REDIRECT_SETTINGS_KEYS:
-            value = settings.get(key)
-            if value in EMPTY_VALUES:
-                continue
-            if key == "breadcrumbs":
-                value = BreadcrumbsField().to_representation(value)
-            filtered_settings[key] = value
-        detail["settings"] = filtered_settings
-
-        self.detail = detail
-
-        LOG.debug(f"redirect {pformat(self.detail)}")
-        # super().__init__ converts every type into strings! do not use it.
-
-    def _add_query_params(self, url):
-        """Change OPDS settings like the frontend does with error.detail."""
-        query_params = {}
-        if isinstance(self.detail, Mapping):
-            settings = self.detail.get("settings", {})
-        else:
-            settings = {}
-        for key in _OPDS_REDIRECT_SETTINGS_KEYS:
-            value = settings.get(key)
-            if value not in EMPTY_VALUES:
-                query_params[key] = value
-        if query_params:
-            url += "?" + urlencode(query_params)
-        return url
-
-    def get_response(self, url_name):
-        """Return a Django Redirect Response."""
-        # only used in codex_exception_handler for opds stuff
-        url = reverse(url_name, kwargs=self.route_kwargs)
-        url = self._add_query_params(url)
-
-        return redirect(url, permanent=False)

codex/librarian/README.md

@@ -1,4 +0,0 @@
-# librarian
-
-Most non-ui tasks are run by the background librariand process. librariand
-spawns threads as well.

codex/librarian/delayed_taskd.py

@@ -1,21 +0,0 @@
-"""Delay tasks."""
-
-from queue import PriorityQueue
-from time import sleep, time
-
-from codex.threads import QueuedThread
-
-
-class DelayedTasksThread(QueuedThread):
-    """Wait for the something before running tasks."""
-
-    def __init__(self, *args, **kwargs):
-        """Use a priority queue."""
-        super().__init__(*args, queue=PriorityQueue(), **kwargs)
-
-    def process_item(self, item):
-        """Sleep and then put tasks on the queue."""
-        delay = max(0.0, item.until - time())
-        sleep(delay)
-        for task in item.tasks:
-            self.librarian_queue.put(task)

codex/librarian/importer/__init__.py

@@ -1 +0,0 @@
-"""Bulk import module."""

codex/librarian/importer/aggregate.py

@@ -1,256 +0,0 @@
-"""Aggregate metadata from comcs to prepare for importing."""
-
-from collections.abc import Mapping
-from pathlib import Path
-
-from codex.librarian.importer.const import (
-    COMIC_FK_FIELD_NAMES,
-    COMIC_M2M_FIELD_NAMES,
-    COMIC_PATHS,
-    COUNT_FIELDS,
-    DICT_MODEL_AGG_MAP,
-    FIS,
-    FKS,
-    FOLDERS_FIELD,
-    GROUP_TREES,
-    ISSUE_COUNT,
-    M2M_MDS,
-    MDS,
-    VOLUME_COUNT,
-)
-from codex.librarian.importer.extract import ExtractMetadataImporter
-from codex.librarian.importer.status import ImportStatusTypes
-from codex.models import Imprint, Publisher, Series, Volume
-from codex.models.admin import AdminFlag
-from codex.status import Status
-from codex.util import max_none
-
-
-class AggregateMetadataImporter(ExtractMetadataImporter):
-    """Aggregate metadata from comics to prepare for importing."""
-
-    _BROWSER_GROUPS = (Publisher, Imprint, Series, Volume)
-    _AGGREGATE_M2M_FIELD_NAMES = (COMIC_M2M_FIELD_NAMES - {"story_arc_numbers"}) | {
-        "story_arcs"
-    }
-
-    @staticmethod
-    def _get_structured_group(md, group_cls, group_field, groups_md):
-        group = md.get(group_field, {})
-        group_name = group.get("name", group_cls.DEFAULT_NAME)
-        count_key = COUNT_FIELDS[group_cls]
-        if count := group.get(count_key):
-            groups_md[count_key] = count
-        return group_name
-
-    @classmethod
-    def _get_group_tree(cls, md):
-        """Create the group tree to counts map for a single comic."""
-        # Create group tree
-        group_tree = []
-        groups_md = {}
-        for group_cls in cls._BROWSER_GROUPS:
-            group_field = group_cls.__name__.lower()
-            if group_cls in COUNT_FIELDS:
-                group_name = cls._get_structured_group(
-                    md, group_cls, group_field, groups_md
-                )
-            else:
-                group_name = md.get(group_field, group_cls.DEFAULT_NAME)
-
-            # This fixes no imprint or whatever being in md
-            md[group_field] = group_name
-            group_tree.append(group_name)
-
-        return {tuple(group_tree): groups_md}
-
-    @classmethod
-    def _get_fk_metadata(cls, md):
-        fk_md = {}
-        for field in COMIC_FK_FIELD_NAMES:
-            if value := md.get(field):
-                fk_md[field] = value
-        return fk_md
-
-    @classmethod
-    def _get_m2m_metadata(cls, md, path):
-        """Many_to_many fields get moved into a separate dict."""
-        m2m_md = {}
-        for field in cls._AGGREGATE_M2M_FIELD_NAMES:
-            if value := md.pop(field, None):
-                m2m_md[field] = value
-        m2m_md[FOLDERS_FIELD] = Path(path).parents
-        return m2m_md
-
-    def _get_path_metadata(self, md, path):
-        """Get the metadata from comicbox and munge it a little."""
-        group_tree_md = self._get_group_tree(md)
-        fk_md = self._get_fk_metadata(md)
-        m2m_md = self._get_m2m_metadata(md, path)
-
-        return md, m2m_md, fk_md, group_tree_md
-
-    @staticmethod
-    def _aggregate_m2m_metadata_dict_value(names, key, values, all_fks):
-        field_name, key_field_name, value_field_name = names
-        if not key:
-            return
-        all_fks[key_field_name].add(key)
-        if key not in all_fks[field_name]:
-            all_fks[field_name][key] = set()
-        if value_field_name:
-            values_set = frozenset(filter(None, values))
-            if values_set:
-                all_fks[field_name][key] |= values_set
-                all_fks[value_field_name] |= values_set
-        else:
-            # Be sure to add Nones to StoryArcNumbers
-            if isinstance(values, Mapping):
-                added_values = tuple(values.items())
-            else:
-                added_values = values
-            all_fks[field_name][key].add(added_values)
-
-    @classmethod
-    def _aggregate_m2m_metadata_dict(cls, field_name, value_dict, all_fks):
-        """Aggregate value dicts."""
-        if not field_name:
-            return
-
-        key_field_name, value_field_name = DICT_MODEL_AGG_MAP[field_name]
-
-        if field_name not in all_fks:
-            all_fks[field_name] = {}
-            all_fks[key_field_name] = set()
-            if value_field_name:
-                all_fks[value_field_name] = set()
-
-        names = field_name, key_field_name, value_field_name
-        for key, values in value_dict.items():
-            cls._aggregate_m2m_metadata_dict_value(names, key, values, all_fks)
-
-    @classmethod
-    def _aggregate_m2m_metadata(cls, all_m2m_mds, m2m_md, all_fks, path):
-        """Aggregate many to many metadata by path."""
-        # m2m fields and fks
-        all_m2m_mds[path] = m2m_md
-        # aggregate fks
-        for field, names in m2m_md.items():
-            if field in DICT_MODEL_AGG_MAP:
-                cls._aggregate_m2m_metadata_dict(field, names, all_fks)
-            elif field != FOLDERS_FIELD:
-                filtered_names = filter(None, names)
-                if not filtered_names:
-                    continue
-                if field not in all_fks:
-                    all_fks[field] = set()
-                all_fks[field] |= frozenset(filtered_names)
-
-    @classmethod
-    def _aggregate_fk_metadata(cls, all_fks, md):
-        """Aggregate non group foreign key metadata."""
-        for field, name in md.items():
-            if field not in all_fks:
-                all_fks[field] = set()
-            all_fks[field].add(name)
-
-    @classmethod
-    def _set_max_group_count(cls, common_args, group_class, index, count_key):
-        """Assign the maximum group count number."""
-        all_fks, group_tree, group_md = common_args
-        group_name = group_tree[0:index]
-        try:
-            count = max_none(
-                all_fks[GROUP_TREES][Series].get(group_name),
-                group_md.get(count_key),
-            )
-        except Exception:
-            count = None
-        all_fks[GROUP_TREES][group_class][group_name] = count
-
-    @classmethod
-    def _aggregate_group_tree_metadata(cls, all_fks, group_tree_md):
-        """Aggregate group tree data by class."""
-        for group_tree, group_md in group_tree_md.items():
-            all_fks[GROUP_TREES][Publisher][group_tree[0:1]] = None
-            all_fks[GROUP_TREES][Imprint][group_tree[0:2]] = None
-            common_args = (all_fks, group_tree, group_md)
-            cls._set_max_group_count(common_args, Series, 3, VOLUME_COUNT)
-            cls._set_max_group_count(common_args, Volume, 4, ISSUE_COUNT)
-
-    def _aggregate_path(self, md, path, status):
-        """Aggregate metadata for one path."""
-        md, m2m_md, fk_md, group_tree_md = self._get_path_metadata(md, path)
-
-        path_str = str(path)
-        if md:
-            all_mds = self.metadata[MDS]
-            all_mds[path_str] = md
-
-        all_fks = self.metadata[FKS]
-        if m2m_md:
-            all_m2m_mds = self.metadata[M2M_MDS]
-            self._aggregate_m2m_metadata(all_m2m_mds, m2m_md, all_fks, path_str)
-
-        if fk_md:
-            self._aggregate_fk_metadata(all_fks, fk_md)
-
-        if group_tree_md:
-            self._aggregate_group_tree_metadata(all_fks, group_tree_md)
-
-        if status:
-            status.complete += 1
-            self.status_controller.update(status)
-
-    def get_aggregate_metadata(
-        self,
-        status=None,
-    ):
-        """Get aggregated metadata for the paths given."""
-        all_paths = self.task.files_modified | self.task.files_created
-        total_paths = len(all_paths)
-
-        if not total_paths:
-            return 0
-        self.log.info(
-            f"Reading tags from {total_paths} comics in {self.library.path}..."
-        )
-        status = Status(ImportStatusTypes.AGGREGATE_TAGS, 0, total_paths)
-        self.status_controller.start(status, notify=False)
-
-        # Set import_metadata flag
-        if self.task.force_import_metadata:
-            import_metadata = True
-        else:
-            key = AdminFlag.FlagChoices.IMPORT_METADATA.value
-            import_metadata = AdminFlag.objects.get(key=key).on
-        if not import_metadata:
-            self.log.warning("Admin flag set to NOT import metadata.")
-
-        # Init metadata, extract and aggregate
-        self.metadata[MDS] = {}
-        self.metadata[M2M_MDS] = {}
-        self.metadata[FKS] = {GROUP_TREES: {cls: {} for cls in self._BROWSER_GROUPS}}
-        self.metadata[FIS] = {}
-        for path in all_paths:
-            md = self.extract_and_clean(path, import_metadata)
-            if md:
-                self._aggregate_path(md, path, status)
-
-        # Aggregate further
-        self.metadata[FKS][COMIC_PATHS] = frozenset(self.metadata[MDS].keys())
-        fis = self.metadata[FIS]
-        self.task.files_modified -= fis.keys()
-        self.task.files_created -= fis.keys()
-
-        # Set statii
-        fi_status = Status(ImportStatusTypes.FAILED_IMPORTS, 0, len(fis))
-        self.status_controller.update(
-            fi_status,
-            notify=False,
-        )
-        count = status.complete if status else 0
-        self.log.info(f"Aggregated tags from {count} comics.")
-
-        self.status_controller.finish(status)
-        return count

codex/librarian/importer/cache.py

@@ -1,90 +0,0 @@
-"""Update Groups timestamp for cover cache busting."""
-
-from django.db.models.aggregates import Count
-from django.db.models.functions.datetime import Now
-from django.db.models.query import Q
-
-from codex.librarian.importer.init import InitImporter
-from codex.librarian.importer.status import ImportStatusTypes
-from codex.models import StoryArc, Volume
-from codex.status import Status
-from codex.views.const import GROUP_MODELS
-
-_UPDATE_FIELDS = ("updated_at",)
-
-
-class CacheUpdateImporter(InitImporter):
-    """Update Groups timestamp for cover cache busting."""
-
-    @staticmethod
-    def _get_update_filter(model, start_time, force_update_group_map):
-        # Get groups with comics updated during this import
-        rel = "storyarcnumber__" if model == StoryArc else ""
-        updated_at_rel = rel + "comic__updated_at__gt"
-        updated_filter = {updated_at_rel: start_time}
-        update_filter = Q(**updated_filter)
-
-        # Get groups with custom covers updated during this import
-        if model != Volume:
-            update_filter |= Q(custom_cover__updated_at__gt=start_time)
-
-        # Get groups to be force updated (usually those with deleted children)
-        if pks := force_update_group_map.get(model):
-            update_filter |= Q(pk__in=pks)
-
-        return update_filter
-
-    @staticmethod
-    def _add_child_count_filter(qs, model):
-        """Filter out groups with no comics."""
-        rel_prefix = "storyarcnumber__" if model == StoryArc else ""
-        rel_prefix += "comic"
-        qs = qs.alias(child_count=Count(f"{rel_prefix}__pk", distinct=True))
-        return qs.filter(child_count__gt=0)
-
-    @classmethod
-    def _update_group_model(cls, force_update_group_map, model, start_time, log_list):
-        """Update a single group model."""
-        update_filter = cls._get_update_filter(
-            model, start_time, force_update_group_map
-        )
-        qs = model.objects.filter(update_filter)
-        qs = cls._add_child_count_filter(qs, model)
-
-        qs = qs.distinct()
-        qs = qs.only(*_UPDATE_FIELDS)
-
-        updated = []
-        for obj in qs:
-            obj.updated_at = Now()
-            updated.append(obj)
-
-        count = len(updated)
-        if count:
-            model.objects.bulk_update(updated, _UPDATE_FIELDS)
-            log_list.append(f"{count} {model.__name__}s")
-        return count
-
-    def update_all_groups(self, force_update_group_map, start_time):
-        """Update timestamps for each group for cover cache busting."""
-        total_count = 0
-        status = Status(ImportStatusTypes.GROUP_UPDATE)
-        self.status_controller.start(status)
-        try:
-            log_list = []
-            for model in GROUP_MODELS:
-                count = self._update_group_model(
-                    force_update_group_map, model, start_time, log_list
-                )
-                if count:
-                    self.log.debug(f"Updated {count} {model.__name__}s timestamps.")
-                status.add_complete(count)
-                self.status_controller.update(status, notify=False)
-                total_count += count
-
-            if total_count:
-                groups_log = ", ".join(log_list)
-                self.log.info(f"Updated timestamps for {groups_log}.")
-                self.changed += total_count
-        finally:
-            self.status_controller.finish(status)