codex 1.7.15__py3-none-any.whl → 1.8.0a2__py3-none-any.whl

codex/librarian/search/tasks.py

@@ -1,37 +0,0 @@
-"""Libarian Tasks for searchd."""
-
-from dataclasses import dataclass
-
-
-@dataclass
-class SearchIndexerTask:
-    """Tasks for the search indexer."""
-
-
-@dataclass
-class SearchIndexUpdateTask(SearchIndexerTask):
-    """Update the search index."""
-
-    rebuild: bool = False
-
-
-@dataclass
-class SearchIndexOptimizeTask(SearchIndexerTask):
-    """Optimize search index."""
-
-    janitor: bool = False
-
-
-@dataclass
-class SearchIndexRemoveStaleTask(SearchIndexerTask):
-    """Remove stale records."""
-
-
-@dataclass
-class SearchIndexAbortTask(SearchIndexerTask):
-    """Abort current search index."""
-
-
-@dataclass
-class SearchIndexClearTask(SearchIndexerTask):
-    """Clear current search index."""

codex/librarian/search/update.py

@@ -1,305 +0,0 @@
-"""Search Index update."""
-
-from datetime import datetime
-from time import time
-from zoneinfo import ZoneInfo
-
-from django.db.models.aggregates import Max
-from django.db.models.expressions import F
-from django.db.models.functions.datetime import Now
-from humanize import intword, naturaldelta
-
-from codex.librarian.search.remove import RemoveMixin
-from codex.librarian.search.status import SearchIndexStatusTypes
-from codex.models import Comic, Library
-from codex.models.comic import ComicFTS
-from codex.models.functions import GroupConcat
-from codex.serializers.fields.browser import CountryField, LanguageField, PyCountryField
-from codex.settings.settings import SEARCH_INDEX_BATCH_SIZE
-from codex.status import Status
-
-_COMICFTS_UPDATE_FIELDS = (
-    # ForeignKeys
-    "publisher",
-    "imprint",
-    "series",
-    "volume",
-    # Attributes
-    "age_rating",
-    "file_type",
-    "country",
-    "issue",
-    "language",
-    "name",
-    "notes",
-    "original_format",
-    "reading_direction",
-    "review",
-    "scan_info",
-    "summary",
-    "tagger",
-    # ManyToMany
-    "characters",
-    "contributors",
-    "genres",
-    "locations",
-    "series_groups",
-    "stories",
-    "story_arcs",
-    "tags",
-    "teams",
-    # Base
-    "updated_at",
-)
-_MIN_UTC_DATE = datetime.min.replace(tzinfo=ZoneInfo("UTC"))
-
-
-class FTSUpdateMixin(RemoveMixin):
-    """Search Index update methods."""
-
-    _STATUS_FINISH_TYPES = (
-        SearchIndexStatusTypes.SEARCH_INDEX_CLEAR,
-        SearchIndexStatusTypes.SEARCH_INDEX_CREATE,
-        SearchIndexStatusTypes.SEARCH_INDEX_UPDATE,
-    )
-
-    def _init_statuses(self, rebuild):
-        """Initialize all statuses order before starting."""
-        statii = []
-        if rebuild:
-            statii += [Status(SearchIndexStatusTypes.SEARCH_INDEX_CLEAR)]
-        else:
-            statii += [Status(SearchIndexStatusTypes.SEARCH_INDEX_REMOVE)]
-        statii += [
-            Status(SearchIndexStatusTypes.SEARCH_INDEX_UPDATE),
-            Status(SearchIndexStatusTypes.SEARCH_INDEX_CREATE),
-        ]
-        self.status_controller.start_many(statii)
-
-    def _update_search_index_clean(self, rebuild):
-        """Clear or clean the search index."""
-        if rebuild:
-            self.log.info("Rebuilding search index...")
-            self.clear_search_index()
-        else:
-            self.remove_stale_records()
-
-    @classmethod
-    def _annotate_fts_query(cls, qs):
-        return qs.annotate(
-            fts_publisher=F("publisher__name"),
-            fts_imprint=F("imprint__name"),
-            fts_series=F("series__name"),
-            fts_volume=F("volume__name"),
-            fts_country=F("country__name"),
-            fts_language=F("language__name"),
-            fts_scan_info=F("scan_info__name"),
-            fts_tagger=F("tagger__name"),
-            fts_characters=GroupConcat("characters__name", distinct=True),
-            fts_contributors=GroupConcat("contributors__person__name", distinct=True),
-            fts_genres=GroupConcat("genres__name", distinct=True),
-            fts_locations=GroupConcat("locations__name", distinct=True),
-            fts_series_groups=GroupConcat("series_groups__name", distinct=True),
-            fts_stories=GroupConcat("stories__name", distinct=True),
-            fts_story_arcs=GroupConcat(
-                "story_arc_numbers__story_arc__name", distinct=True
-            ),
-            fts_tags=GroupConcat("tags__name", distinct=True),
-            fts_teams=GroupConcat("teams__name", distinct=True),
-        )
-
-    @staticmethod
-    def _get_pycountry_fts_field(field_instance: PyCountryField, iso_code):
-        if not iso_code:
-            return ""
-        return ",".join((iso_code, field_instance.to_representation(iso_code)))
-
-    def _get_comics_fts_list(
-        self, comics, country_field, language_field, obj_list, create
-    ):
-        """Prepare a batch of search entries."""
-        for comic in comics:
-            country = self._get_pycountry_fts_field(country_field, comic.fts_country)
-            language = self._get_pycountry_fts_field(language_field, comic.fts_language)
-
-            now = Now()
-            comicfts = ComicFTS(
-                comic_id=comic.pk,
-                updated_at=now,
-                publisher=comic.fts_publisher,
-                imprint=comic.fts_imprint,
-                series=comic.fts_series,
-                volume=comic.fts_volume,
-                issue=comic.issue,
-                name=comic.name,
-                age_rating=comic.age_rating,
-                country=country,
-                file_type=comic.file_type,
-                language=language,
-                notes=comic.notes,
-                original_format=comic.original_format,
-                reading_direction=comic.reading_direction,
-                review=comic.review,
-                scan_info=comic.fts_scan_info,
-                summary=comic.summary,
-                tagger=comic.fts_tagger,
-                # ManyToMany
-                characters=comic.fts_characters,
-                contributors=comic.fts_contributors,
-                genres=comic.fts_genres,
-                locations=comic.fts_locations,
-                series_groups=comic.fts_series_groups,
-                stories=comic.fts_stories,
-                story_arcs=comic.fts_story_arcs,
-                tags=comic.fts_tags,
-                teams=comic.fts_teams,
-            )
-            if create:
-                comicfts.created_at = now
-            obj_list.append(comicfts)
-            self.log.debug(f"{len(obj_list)}/{comics.count()} entries prepped.")
-
-    def _get_comicfts_list(self, comics, create: bool):
-        """Create a ComicFTS object for bulk_create or bulk_update."""
-        country_field = CountryField()
-        language_field = LanguageField()
-        obj_list = []
-        comics = self._annotate_fts_query(comics)
-        self._get_comics_fts_list(
-            comics, country_field, language_field, obj_list, create
-        )
-        return obj_list
-
-    def _update_search_index_finish(self, count, verb, status):
-        verb = verb.capitalize() + "d"
-        if count:
-            self.log.info(f"{verb} {count} search entries.")
-        else:
-            self.log.debug(f"{verb} no search entries.")
-        self.status_controller.finish(status)
-
-    def _update_search_index_operate(self, comics_qs, create: bool):
-        count = comics_qs.count()
-        verb = "create" if create else "update"
-        if not count:
-            self.log.info(f"No search entries to {verb}.")
-
-        if count > SEARCH_INDEX_BATCH_SIZE:
-            human_size = intword(SEARCH_INDEX_BATCH_SIZE)
-            self.log.debug(
-                f"Batching this search engine {verb} operation in to chunks of {human_size}."
-                f" Search engine {verb}s run much faster as one large batch but then there's no progress updates."
-                " You may adjust the batch size with the environment variable CODEX_SEARCH_INDEX_BATCH_SIZE."
-            )
-            subtitle = f"Chunks of {human_size}"
-        else:
-            subtitle = ""
-
-        status_type = (
-            SearchIndexStatusTypes.SEARCH_INDEX_CREATE
-            if create
-            else SearchIndexStatusTypes.SEARCH_INDEX_UPDATE
-        )
-        operate_status = Status(status_type, total=count, subtitle=subtitle)
-        self.status_controller.start(operate_status)
-
-        batch_from = 0
-        try:
-            while batch_from < count:
-                batch_start = time()
-                batch_to = batch_from + SEARCH_INDEX_BATCH_SIZE
-                comics_batch = comics_qs[batch_from:batch_to]
-                self.log.debug(
-                    f"Checking {comics_batch.count()}/{count} comics for search index operations..."
-                )
-                if self.abort_event.is_set():
-                    break
-                operate_comicfts = self._get_comicfts_list(comics_batch, create)
-                operate_comicfts_count = len(operate_comicfts)
-                verbing = (verb[:-1] + "ing").capitalize()
-                self.log.debug(f"{verbing} {operate_comicfts_count} search entries...")
-                if self.abort_event.is_set():
-                    break
-                if operate_status.complete is None:
-                    operate_status.add_complete(0)
-                    self.status_controller.update(operate_status, notify=True)
-                if create:
-                    ComicFTS.objects.bulk_create(operate_comicfts)
-                else:
-                    ComicFTS.objects.bulk_update(
-                        operate_comicfts, _COMICFTS_UPDATE_FIELDS
-                    )
-                operate_status.add_complete(operate_comicfts_count)
-                self.status_controller.update(operate_status)
-                batch_time = time() - batch_start
-                eps = round(count / batch_time)
-                verbed = (verb + "d").capitalize()
-                self.log.debug(
-                    f"{verbed} {operate_comicfts_count}/{count} search entries in {batch_time}, {eps} per second."
-                )
-                batch_from = batch_to
-        finally:
-            self._update_search_index_finish(count, verb, operate_status)
-
-    def _update_search_index_update(self, all_indexed_comic_ids):
-        """Update out of date search entries."""
-        fts_watermark = ComicFTS.objects.aggregate(max=Max("updated_at"))["max"]
-        if not fts_watermark:
-            fts_watermark = _MIN_UTC_DATE
-            since = "the fracturing of the multiverse"
-        else:
-            since = fts_watermark
-        self.log.info(f"Looking for search entries to update since {since}...")
-        out_of_date_comics = Comic.objects.filter(
-            pk__in=all_indexed_comic_ids, updated_at__gt=fts_watermark
-        )
-        self._update_search_index_operate(out_of_date_comics, create=False)
-
-    def _update_search_index_create(self, all_indexed_comic_ids):
-        """Create missing search entries."""
-        self.log.info("Looking for missing search entries to create...")
-        missing_comics = Comic.objects.all()
-        if len(all_indexed_comic_ids):
-            missing_comics = missing_comics.exclude(pk__in=all_indexed_comic_ids)
-        self._update_search_index_operate(missing_comics, create=True)
-
-    def _update_search_index(self, start_time, rebuild):
-        """Update or Rebuild the search index."""
-        any_update_in_progress = Library.objects.filter(
-            covers_only=False, update_in_progress=True
-        ).exists()
-        if any_update_in_progress:
-            self.log.debug(
-                "Database update in progress, not updating search index yet."
-            )
-            return
-
-        self._init_statuses(rebuild)
-
-        if self.abort_event.is_set():
-            return
-        self._update_search_index_clean(rebuild)
-        if self.abort_event.is_set():
-            return
-        all_indexed_comic_ids = ComicFTS.objects.values_list("comic_id", flat=True)
-        if self.abort_event.is_set():
-            return
-        self._update_search_index_update(all_indexed_comic_ids)
-        if self.abort_event.is_set():
-            return
-        self._update_search_index_create(all_indexed_comic_ids)
-
-        elapsed_time = time() - start_time
-        elapsed = naturaldelta(elapsed_time)
-        self.log.info(f"Search index updated in {elapsed}.")
-
-    def update_search_index(self, rebuild: bool):
-        """Update or Rebuild the search index."""
-        start_time = time()
-        self.abort_event.clear()
-        try:
-            self._update_search_index(start_time, rebuild)
-        except Exception:
-            self.log.exception("Update search index")
-        finally:
-            self.status_controller.finish_many(self._STATUS_FINISH_TYPES)

codex/librarian/watchdog/README.md

@@ -1,11 +0,0 @@
-# Watchdog
-
-- Extends the Watchdog API with custom Observers, DirSnapshot, Emitter.
-- The Observers decide to watch different paths by querying the Library objects.
-- A CodexDatabaseSnapshot exists to compare the codex database against a
-  standard DirSnapshot
-- Events batch into large tasks for the Updater by the EventBatcher. The Emitter
-  could do this more efficiently, but I'm also using the standard
-  FileSystemObserver that spits them into a queue.
-
-[Watchdog project page](https://github.com/gorakhargosh/watchdog)

codex/logger/__init__.py

@@ -1 +0,0 @@
-"""Central logger thread."""

codex/logger/formatter.py

@@ -1,36 +0,0 @@
-"""Logging Color Formatter."""
-
-import logging
-from types import MappingProxyType
-
-from colors import color
-
-
-class ColorFormatter(logging.Formatter):
-    """Logging Formatter to add colors and count warning / errors."""
-
-    FORMAT_COLORS = MappingProxyType(
-        {
-            "CRITICAL": {"fg": "red", "style": "bold"},
-            "ERROR": {"fg": "red"},
-            "WARNING": {"fg": "yellow"},
-            "INFO": {"fg": "green"},
-            "DEBUG": {"fg": "black", "style": "bold"},
-            "NOTSET": {"fg": "blue"},
-        }
-    )
-
-    def __init__(self, fmt, **kwargs):
-        """Set up the FORMATS dict."""
-        super().__init__(**kwargs)
-        self.formatters = {}
-        for level_name, args in self.FORMAT_COLORS.items():
-            levelno = getattr(logging, level_name)
-            template = color(fmt, **args)
-            formatter = logging.Formatter(fmt=template, **kwargs)
-            self.formatters[levelno] = formatter
-
-    def format(self, record):
-        """Format each log message."""
-        formatter = self.formatters[record.levelno]
-        return formatter.format(record)

codex/logger/logger.py

@@ -1,46 +0,0 @@
-"""Logging classes."""
-
-import logging
-from contextlib import suppress
-from logging.handlers import QueueHandler
-
-from codex.logger.mp_queue import LOG_QUEUE
-from codex.settings.settings import LOGLEVEL
-
-
-class CodexQueueHandler(QueueHandler):
-    """Special QueueHandler that removes unpickable elements."""
-
-    def enqueue(self, record):
-        """Remove unserializable element before queueing."""
-        with suppress(KeyError):
-            # Fixes _pickle.PicklingError: Cannot pickle ResolverMatch.
-            # Somewhere inside request is this unpicklable object.
-            record.__dict__["request"].resolver_match = None
-        super().enqueue(record)
-
-
-_LOG_HANDLER = CodexQueueHandler(LOG_QUEUE)
-
-
-def _ensure_handler(logger, queue):
-    """Ensure we only have one correct handler."""
-    has_correct = False
-    for handler in tuple(logger.handlers):
-        if not has_correct and getattr(handler, "queue", None) == queue:
-            has_correct = True
-        else:
-            logger.removeHandler(handler)
-
-    if not logger.handlers:
-        handler = _LOG_HANDLER if queue == LOG_QUEUE else CodexQueueHandler(queue)
-        logger.addHandler(handler)
-
-
-def get_logger(name=None, queue=LOG_QUEUE):
-    """Get the logger with only one handler."""
-    logger = logging.getLogger(name)
-    logger.setLevel(LOGLEVEL)
-    logger.propagate = False
-    _ensure_handler(logger, queue)
-    return logger

codex/logger/loggerd.py

@@ -1,102 +0,0 @@
-"""Central Logging Thread."""
-
-import logging
-import shutil
-from logging.handlers import QueueListener, RotatingFileHandler
-from lzma import LZMAFile
-from pathlib import Path
-
-from codex.logger.formatter import ColorFormatter
-from codex.settings.settings import DEBUG, LOG_DIR, LOG_TO_CONSOLE, LOG_TO_FILE
-
-_LOG_FMT = "{asctime} {levelname:7} {message}"
-_DEBUG_LOG_FMT = "{asctime} {levelname:7} {name:25} {message}"
-_DATEFMT = "%Y-%m-%d %H:%M:%S %Z"
-_FORMATTER_STYLE = "{"
-_LOG_PATH = LOG_DIR / "codex.log"
-_LOG_MAX_BYTES = 10 * 2**20  # 10 MiB
-_LOG_BACKUP_COUNT = 30
-
-
-class RotatingXZFileHandler(RotatingFileHandler):
-    """Rotating File Handler rotates into XZ."""
-
-    @staticmethod
-    def namer(name):
-        """Name files with xz."""
-        return name + ".xz"
-
-    @staticmethod
-    def rotator(source, dest):
-        """Compress File."""
-        source_path = Path(source)
-        with (
-            source_path.open("rb") as f_in,
-            LZMAFile(dest, mode="wb", preset=9) as f_out,
-        ):
-            shutil.copyfileobj(f_in, f_out)
-        source_path.unlink()
-
-
-class CodexLogQueueListener(QueueListener):
-    """Host for logging queue listener."""
-
-    @classmethod
-    def _get_file_log_handler(cls, fmt):
-        """Get the log handlers for initialization."""
-        handler = None
-        try:
-            _LOG_PATH.parent.mkdir(exist_ok=True, parents=True)
-            handler = RotatingXZFileHandler(
-                _LOG_PATH,
-                maxBytes=_LOG_MAX_BYTES,
-                backupCount=_LOG_BACKUP_COUNT,
-                delay=True,
-            )
-            formatter = logging.Formatter(fmt, style=_FORMATTER_STYLE, datefmt=_DATEFMT)
-            handler.setFormatter(formatter)
-        except Exception as exc:
-            print("ERROR creating file logging handler", exc)
-        return handler
-
-    @classmethod
-    def _get_console_handler(cls, fmt):
-        """Create the console handler."""
-        handler = None
-        try:
-            handler = logging.StreamHandler()
-            formatter = ColorFormatter(fmt, style=_FORMATTER_STYLE, datefmt=_DATEFMT)
-            handler.setFormatter(formatter)
-        except Exception as exc:
-            print("ERROR creating console logging handler", exc)
-        return handler
-
-    @classmethod
-    def _get_log_handlers(cls):
-        """Get handlers."""
-        handlers = []
-        fmt = _DEBUG_LOG_FMT if DEBUG else _LOG_FMT
-
-        if LOG_TO_FILE:
-            handlers.append(cls._get_file_log_handler(fmt))
-        if LOG_TO_CONSOLE:
-            handlers.append(cls._get_console_handler(fmt))
-        return handlers
-
-    def __init__(self, log_queue):
-        """Start self with handlers."""
-        handlers = self._get_log_handlers()
-        self.log_queue = log_queue
-        super().__init__(log_queue, *handlers)
-
-    def stop(self):
-        """Stop listener and cleans up handlers."""
-        super().stop()
-        for handler in self.handlers:
-            handler.flush()
-            handler.close()
-        self.handlers = ()
-        while not self.log_queue.empty():
-            self.log_queue.get_nowait()
-        self.log_queue.close()
-        self.log_queue.join_thread()

codex/logger/mp_queue.py

@@ -1,5 +0,0 @@
-"""Central logging queue."""
-
-from multiprocessing import Queue
-
-LOG_QUEUE = Queue()

codex/logger_base.py

@@ -1,12 +0,0 @@
-"""Class to run librarian tasks inline without a thread."""
-
-from codex.logger.logger import get_logger
-
-
-class LoggerBaseMixin:
-    """A class that holds it's own logger."""
-
-    def init_logger(self, log_queue):
-        """Set up logger."""
-        self.log_queue = log_queue
-        self.log = get_logger(self.__class__.__name__, log_queue)

codex/memory.py

@@ -1,56 +0,0 @@
-"""Detect how much memory we're working with."""
-
-import resource
-from contextlib import suppress
-from pathlib import Path
-
-from psutil import virtual_memory
-
-# cgroups1
-MEMORY_STAT_PATH = "/sys/fs/cgroup/memory/memory.stat"
-
-# cgroups2
-MEMORY_MAX_PATH = "/sys/fs/cgroup/memory.max"
-
-DIVISORS = {"b": 1, "k": 1024, "m": 1024**2, "g": 1024**3}
-
-
-def get_cgroups2_mem_limit():
-    """Get mem limit from cgroups2."""
-    with Path(MEMORY_MAX_PATH).open("r") as limit:
-        return int(limit.read())
-
-
-def get_cgroups1_mem_limit():
-    """Get mem limit from cgroups1."""
-    with Path(MEMORY_STAT_PATH).open("r") as mem_stat_file:
-        for line in mem_stat_file:
-            parts = line.split()
-            if not parts or len(parts) < 2:  # noqa: PLR2004
-                continue
-            if "hierarchical_memory_limit" in parts[0]:
-                return int(parts[1])
-    return None
-
-
-def get_mem_limit(divisor="b"):
-    """
-    Get the current memlimit.
-
-    If we're in a container set the limit too.
-    """
-    mem_limit = None
-    api_funcs = (get_cgroups2_mem_limit, get_cgroups1_mem_limit)
-    for func in api_funcs:
-        with suppress(Exception):
-            mem_limit = func()
-        if mem_limit:
-            break
-    if mem_limit:
-        # Set the process memlimit.
-        resource.setrlimit(resource.RLIMIT_AS, (mem_limit, mem_limit))
-
-    if not mem_limit:
-        # get the raw host memory
-        mem_limit = virtual_memory().total
-    return mem_limit / DIVISORS.get(divisor, 1)

codex/permissions.py

@@ -1,19 +0,0 @@
-"""Codex drf permissions."""
-
-from rest_framework.permissions import BasePermission, IsAdminUser
-
-from codex.models import Timestamp
-
-
-class HasAPIKeyOrIsAdminUser(BasePermission):
-    """Does the request have the current api key."""
-
-    def has_permission(self, request, view):
-        """Test the request api key against the database."""
-        data = request.GET if request.method == "GET" else request.POST
-        api_key = data.get("apiKey")
-        if not api_key:
-            return IsAdminUser().has_permission(request, view)
-        return Timestamp.objects.filter(
-            key=Timestamp.TimestampChoices.API_KEY.value, version=api_key
-        ).exists()

codex/registration.py

@@ -1,16 +0,0 @@
-"""Patch settings at runtime."""
-
-from rest_registration.settings import registration_settings
-
-from codex.models import AdminFlag
-
-
-def patch_registration_setting():
-    """Patch rest_registration setting."""
-    # Technically this is a no-no, but rest-registration makes it easy.
-    enr = (
-        AdminFlag.objects.only("on")
-        .get(key=AdminFlag.FlagChoices.REGISTRATION.value)
-        .on
-    )
-    registration_settings.user_settings["REGISTER_FLOW_ENABLED"] = enr

codex/serializers/README.md

@@ -1,4 +0,0 @@
-# serializers
-
-Django Rest Framework serializers.
-[DRF API Guide](https://www.django-rest-framework.org/api-guide/serializers/).

codex/settings/README.md

@@ -1,4 +0,0 @@
-# codex.settings
-
-I moved the functions that service the django settings file into their own
-module to clean up the readability of the settings file.