codex 1.6.19__py3-none-any.whl → 1.7.0a0__py3-none-any.whl

codex/search/backend_search.py

@@ -1,229 +0,0 @@
-"""Backend search that gets the reverse variable as a parameter."""
-
-import warnings
-from datetime import date, datetime
-
-from django.conf import settings
-from django.utils.encoding import force_str
-
-# Bubble up the correct error.
-from whoosh.sorting import Count, DateRangeFacet, FieldFacet
-from whoosh.support.relativedelta import relativedelta as RelativeDelta
-
-from codex._vendor.haystack.backends import log_query
-from codex._vendor.haystack.constants import DJANGO_CT
-from codex._vendor.haystack.exceptions import SearchBackendError
-from codex._vendor.haystack.utils import get_model_ct
-
-
-class CodexBackendSearchMixin:
-    @log_query
-    def search(
-        self,
-        query_string,
-        sort_by=None,
-        start_offset=0,
-        end_offset=None,
-        fields="",
-        highlight=False,
-        facets=None,
-        date_facets=None,
-        query_facets=None,
-        narrow_queries=None,
-        spelling_query=None,
-        within=None,
-        dwithin=None,
-        distance_point=None,
-        models=None,
-        limit_to_registered_models=None,
-        result_class=None,
-        reverse=False,
-        **kwargs,
-    ):
-        if not self.setup_complete:
-            self.setup()
-
-        # A zero length query should return no results.
-        if len(query_string) == 0:
-            return {"results": [], "hits": 0}
-
-        query_string = force_str(query_string)
-
-        # A one-character query (non-wildcard) gets nabbed by a stopwords
-        # filter and should yield zero results.
-        if len(query_string) <= 1 and query_string != "*":
-            return {"results": [], "hits": 0}
-
-        # reverse = False # old reverse flag
-
-        if sort_by is not None:
-            # Determine if we need to reverse the results and if Whoosh can
-            # handle what it's being asked to sort by. Reversing is an
-            # all-or-nothing action, unfortunately.
-            sort_by_list = []
-            reverse_counter = 0
-
-            for order_by in sort_by:
-                if order_by.startswith("-"):
-                    reverse_counter += 1
-            if reverse_counter and reverse_counter != len(sort_by):
-                raise SearchBackendError(
-                    "Whoosh requires all order_by fields"
-                    " to use the same sort direction"
-                )
-
-            for order_by in sort_by:
-                if order_by.startswith("-"):
-                    sort_by_list.append(order_by[1:])
-
-                    if len(sort_by_list) == 1:
-                        reverse = True
-                else:
-                    sort_by_list.append(order_by)
-
-                    if len(sort_by_list) == 1:
-                        reverse = False
-
-            sort_by = sort_by_list
-
-        group_by = []
-        facet_types = {}
-        if facets is not None:
-            group_by += [
-                FieldFacet(facet, allow_overlap=True, maptype=Count) for facet in facets
-            ]
-            facet_types.update({facet: "fields" for facet in facets})
-
-        if date_facets is not None:
-
-            def _fixup_datetime(dt):
-                if isinstance(dt, datetime):
-                    return dt
-                if isinstance(dt, date):
-                    return datetime(dt.year, dt.month, dt.day)
-                raise ValueError
-
-            for key, value in date_facets.items():
-                start = _fixup_datetime(value["start_date"])
-                end = _fixup_datetime(value["end_date"])
-                gap_by = value["gap_by"]
-                gap_amount = value.get("gap_amount", 1)
-                gap = RelativeDelta(**{"%ss" % gap_by: gap_amount})
-                group_by.append(DateRangeFacet(key, start, end, gap, maptype=Count))
-                facet_types[key] = "dates"
-
-        if query_facets is not None:
-            warnings.warn(
-                "Whoosh does not handle query faceting.", Warning, stacklevel=2
-            )
-
-        narrowed_results = None
-        self.index = self.index.refresh()
-
-        if limit_to_registered_models is None:
-            limit_to_registered_models = getattr(
-                settings, "HAYSTACK_LIMIT_TO_REGISTERED_MODELS", True
-            )
-
-        if models and len(models):
-            model_choices = sorted(get_model_ct(model) for model in models)
-        elif limit_to_registered_models:
-            # Using narrow queries, limit the results to only models handled
-            # with the current routers.
-            model_choices = self.build_models_list()
-        else:
-            model_choices = []
-
-        if len(model_choices) > 0:
-            if narrow_queries is None:
-                narrow_queries = set()
-
-            narrow_queries.add(
-                " OR ".join(["%s:%s" % (DJANGO_CT, rm) for rm in model_choices])
-            )
-
-        narrow_searcher = None
-
-        if narrow_queries is not None:
-            # Potentially expensive? I don't see another way to do it in Whoosh...
-            narrow_searcher = self.index.searcher()
-
-            for nq in narrow_queries:
-                recent_narrowed_results = narrow_searcher.search(
-                    self.parser.parse(force_str(nq)), limit=None
-                )
-
-                if len(recent_narrowed_results) <= 0:
-                    return {"results": [], "hits": 0}
-
-                if narrowed_results is not None:
-                    narrowed_results.filter(recent_narrowed_results)
-                else:
-                    narrowed_results = recent_narrowed_results
-
-        self.index = self.index.refresh()
-
-        if self.index.doc_count():
-            searcher = self.index.searcher()
-            parsed_query = self.parser.parse(query_string)
-
-            # In the event of an invalid/stopworded query, recover gracefully.
-            if parsed_query is None:
-                return {"results": [], "hits": 0}
-
-            page_num, page_length = self.calculate_page(start_offset, end_offset)
-
-            search_kwargs = {
-                "pagelen": page_length,
-                "sortedby": sort_by,
-                "reverse": reverse,
-                "groupedby": group_by,
-            }
-
-            # Handle the case where the results have been narrowed.
-            if narrowed_results is not None:
-                search_kwargs["filter"] = narrowed_results
-
-            try:
-                raw_page = searcher.search_page(parsed_query, page_num, **search_kwargs)
-            except ValueError:
-                if not self.silently_fail:
-                    raise
-
-                return {"results": [], "hits": 0, "spelling_suggestion": None}
-
-            # Because as of Whoosh 2.5.1, it will return the wrong page of
-            # results if you request something too high. :(
-            if raw_page.pagenum < page_num:
-                return {"results": [], "hits": 0, "spelling_suggestion": None}
-
-            results = self._process_results(
-                raw_page,
-                highlight=highlight,
-                query_string=query_string,
-                spelling_query=spelling_query,
-                result_class=result_class,
-                facet_types=facet_types,
-            )
-            searcher.close()
-
-            if hasattr(narrow_searcher, "close"):
-                narrow_searcher.close()
-
-            return results
-        else:
-            if self.include_spelling:
-                if spelling_query:
-                    spelling_suggestion = self.create_spelling_suggestion(
-                        spelling_query
-                    )
-                else:
-                    spelling_suggestion = self.create_spelling_suggestion(query_string)
-            else:
-                spelling_suggestion = None
-
-            return {
-                "results": [],
-                "hits": 0,
-                "spelling_suggestion": spelling_suggestion,
-            }

codex/search/engine.py

@@ -1,33 +0,0 @@
-"""Custom Haystack Search Engine."""
-
-from codex._vendor.haystack.backends import UnifiedIndex
-from codex._vendor.haystack.backends.whoosh_backend import (
-    WhooshEngine,
-)
-from codex.search.backend import CodexSearchBackend
-from codex.search.indexes import ComicIndex
-from codex.search.query import CodexSearchQuery
-
-
-class CodexUnifiedIndex(UnifiedIndex):
-    """Custom Codex Unified Index."""
-
-    def collect_indexes(self):
-        """Replace auto app.search_index finding with one exact instance."""
-        # Because I moved search_indexes into codex.search
-        return [ComicIndex()]
-
-
-class CodexSearchEngine(WhooshEngine):
-    """A search engine with a locking backend."""
-
-    backend = CodexSearchBackend
-    query = CodexSearchQuery
-    unified_index = CodexUnifiedIndex
-
-    def __init__(self, using=None, queue_kwargs=None):
-        """Initialize options with queue kwargs."""
-        super().__init__(using=using)
-        if not queue_kwargs:
-            queue_kwargs = {}
-        self.options.update(queue_kwargs)

codex/search/indexes.py

@@ -1,79 +0,0 @@
-"""Haystack search indexes."""
-
-from codex._vendor.haystack.fields import CharField, FloatField, IntegerField
-from codex._vendor.haystack.indexes import Indexable, ModelSearchIndex
-from codex.models import Comic
-
-
-class ComicIndex(ModelSearchIndex, Indexable):
-    """Search index for the Comic model."""
-
-    # Groups
-    publisher = CharField(model_attr="publisher__name")
-    imprint = CharField(model_attr="imprint__name")
-    series = CharField(model_attr="series__name", boost=1.125)
-    volume = IntegerField(model_attr="volume__name", null=True)
-
-    # Related Fields
-    age_rating = CharField(model_attr="age_rating__name", null=True)
-    country = CharField(model_attr="country__name", null=True)
-    language = CharField(model_attr="language__name", null=True)
-    original_format = CharField(model_attr="original_format__name", null=True)
-    scan_info = CharField(model_attr="scan_info__name", null=True)
-    tagger = CharField(model_attr="tagger__name", null=True)
-
-    # Many to Many Fields
-    characters = CharField(model_attr="characters__name", null=True)
-    contributors = CharField(model_attr="contributors__person__name", null=True)
-    genres = CharField(model_attr="genres__name", null=True)
-    locations = CharField(model_attr="locations__name", null=True)
-    identifier = CharField(model_attr="identifiers__nss", null=True)
-    identifier_type = CharField(
-        model_attr="identifiers__identifier_type__name", null=True
-    )
-    series_groups = CharField(model_attr="series_groups__name", null=True)
-    stories = CharField(model_attr="stories__name", null=True)
-    story_arcs = CharField(
-        model_attr="story_arc_numbers__story_arc__name",
-        null=True,
-    )
-    tags = CharField(model_attr="tags__name", null=True)
-    teams = CharField(model_attr="teams__name", null=True)
-
-    # This makes issue field searching exact rather than fuzzy
-    issue = CharField(model_attr="issue", null=True)
-
-    # Floats are a better search experience than fixed decimals
-    issue_number = FloatField(model_attr="issue_number", null=True)
-    community_rating = FloatField(model_attr="community_rating", null=True)
-    critical_rating = FloatField(model_attr="critical_rating", null=True)
-
-    search_path = CharField(model_attr="search_path", null=True)
-
-    class Meta:
-        """Model & field include list."""
-
-        model = Comic
-        fields = (
-            "created_at",
-            "day",
-            "date",
-            "decade",
-            "issue_suffix",
-            "month",
-            "monochrome",
-            "name",
-            "notes",
-            "page_count",
-            "reading_direction",
-            "review",
-            "size",
-            "summary",
-            "updated_at",
-            "user_rating",
-            "year",
-        )
-
-    def get_updated_field(self):  # type: ignore
-        """To only update models that have changed."""
-        return "updated_at"

codex/search/query.py

@@ -1,67 +0,0 @@
-"""Custom Codex search query classes."""
-
-from codex._vendor.haystack.backends.whoosh_backend import (
-    WhooshSearchQuery,
-)
-from codex._vendor.haystack.constants import DJANGO_ID
-from codex._vendor.haystack.query import SearchQuerySet
-
-
-class CodexSearchQuery(WhooshSearchQuery):
-    """Custom search qeuery."""
-
-    def __init__(self, *args, **kwargs):
-        """Give CodexSearchQuery a reverse attribute."""
-        super().__init__(*args, **kwargs)
-        self._reverse: bool = False
-        self._filter_comic_ids: frozenset | None = None
-
-    def clean(self, query_fragment):
-        """Optimize to noop because RESERVED_ consts are empty in the backend."""
-        return query_fragment
-
-    def order_reverse(self):
-        """Invert the reverse attribute."""
-        self._reverse = not self._reverse
-
-    def filter_comic_ids(self, ids):
-        """Set a filter for the query."""
-        self._filter_comic_ids = frozenset(ids)
-
-    def get_results(self, **kwargs):
-        """Send custom get results the reverse flag."""
-        custom_params = {}
-        if self._reverse:
-            custom_params["reverse"] = self._reverse
-
-        if self._filter_comic_ids:
-            filter_query = " OR ".join(
-                [f"{DJANGO_ID}:{pk}" for pk in self._filter_comic_ids]
-            )
-            custom_params["narrow_queries"] = {filter_query}
-
-        return super().get_results(**custom_params, **kwargs)
-
-
-class CodexSearchQuerySet(SearchQuerySet):
-    """SearchQuerySet with order_reverse method."""
-
-    def order_reverse(self):
-        """Alters the order in which the results should appear."""
-        clone = self._clone()
-        clone.query.order_reverse()  # type: ignore
-        return clone
-
-    def filter_comic_ids(self, ids):
-        """Filter search by comic ids."""
-        clone = self._clone()
-        clone.query.filter_comic_ids(ids)  # type: ignore
-        return clone
-
-    def values_list(self, *args, **kwargs):
-        """Copy reverse setting into CodexSearchQuery."""
-        clone = super().values_list(*args, **kwargs)
-        if clone.query and self.query:
-            clone.query._reverse = self.query._reverse  # noqa: SLF001
-            clone.query._filter_comic_ids = self.query._filter_comic_ids  # noqa: SLF001
-        return clone

codex/search/writing.py

@@ -1,180 +0,0 @@
-"""Custom Codex Writer."""
-
-from threading import RLock
-from time import sleep
-
-from whoosh.index import FileIndex, LockError
-from whoosh.reading import MultiReader
-from whoosh.writing import BufferedWriter
-
-
-class AbortOperationError(Exception):
-    """Interrupt the operation because something more important happened."""
-
-
-class CodexWriter(BufferedWriter):
-    """MP safe Buffered Writer that locks the index writer much more granularly.
-
-    XXX The period and limit args are ignored and this writer behaves more like the
-    AsycWriter. The AsyncWriter could possibly be extended to not lock until commit
-    time like this one and use less code.
-    """
-
-    # For waiting for the writer lock
-    _DELAY = 0.25
-
-    def __init__(self, index, writerargs=None, commitargs=None):
-        """Initialize with special values."""
-        # Same as BufferedWriter init without caching the index.writer() as self.writer
-        self.index = index
-        self.writerargs = writerargs or {}
-        self.commitargs = commitargs or {}
-
-        self.lock = RLock()
-
-        self._make_ram_index()
-
-        # Caching the schema so it doesn't need to derive from a writer every time.
-        self._schema = None
-        # Debug dict to print on close to diagnose what's waiting for locks.
-        self._time_sleeping = {}
-
-    def get_writer(self, caller="unknown"):
-        """Wait for the lock to be available and furnish the writer."""
-        writer = None
-        while writer is None:
-            try:
-                writer = self.index.refresh().writer(**self.writerargs)
-            except LockError:
-                sleep(self._DELAY)
-                if caller not in self._time_sleeping:
-                    self._time_sleeping[caller] = 0
-                self._time_sleeping[caller] += self._DELAY
-        return writer
-
-    @property
-    def schema(self):
-        """Get a cached schema."""
-        if not self._schema:
-            info = self.index.refresh()._read_toc()  # noqa SLF001
-            self._schema = info.schema
-        return self._schema
-
-    def _get_non_locking_writer_reader(self):
-        """Get self.writer.reader() without locking.
-
-        Copied from SegmentWriter.reader()
-
-        XXX Runs the risk of being slightly out of date if it's actively being
-        written by other procs.
-        """
-        with self.lock:
-            ramreader = self._get_ram_reader()
-
-        index = self.index.refresh()
-        info = index._read_toc()  # noqa SLF001
-        return FileIndex._reader(  # noqa SLF001
-            index.storage,
-            info.schema,
-            info.segments,
-            info.generation + 1,
-            # using the ram index for reuse massively reduces
-            # duplication, but is a hack.
-            reuse=ramreader,
-        )
-
-    def reader(self, **_kwargs):
-        """Get the reader without locking the writer."""
-        reader = self._get_non_locking_writer_reader()
-
-        with self.lock:
-            ramreader = self._get_ram_reader()
-
-        # If there are in-memory docs, combine the readers
-        if ramreader.doc_count():
-            if reader.is_atomic():
-                reader = MultiReader([reader, ramreader])
-            else:
-                reader.add_reader(ramreader)  # type: ignore
-
-        return reader
-
-    def commit(self, restart=True, reader=None):  # noqa: ARG002
-        """Commit with a writer we get now."""
-        with self.lock:
-            writer = self.get_writer("commit")
-            if reader:
-                writer.add_reader(reader)
-            ramreader = self._get_ram_reader()
-            writer.add_reader(ramreader)
-            writer.commit(**self.commitargs)
-
-    def add_reader(self, reader):
-        """Do a commit with the supplied reader."""
-        # Pass through to the underlying on-disk index
-        self.commit(reader=reader)
-
-    def add_document(self, **fields):
-        """Add a document with the cached schema."""
-        with self.lock, self.codec.writer(self.schema) as w:
-            # Hijack a writer to make the calls into the ram index.
-            w.add_document(**fields)
-
-    def _stage_segment_deletes(self, writer, docnums):
-        """Get the deletes to perform."""
-        base = self.index.refresh().doc_count_all()
-        count = 0
-        with self.lock:
-            for docnum in sorted(docnums):
-                if docnum < base:
-                    segment, segdocnum = writer._segment_and_docnum(  # noqa SLF001
-                        docnum
-                    )
-                else:
-                    segment = self.codec.segment
-                    segdocnum = docnum - base
-                segment.delete_document(segdocnum, delete=True)
-                count += 1
-        return count
-
-    def delete_batch_documents(self, docnums):
-        """Delete multiple documents very quickly."""
-        count = 0
-        writer = self.get_writer("delete_batch_docnums")
-        count = self._stage_segment_deletes(writer, docnums)
-
-        # Commit
-        try:
-            writer.commit(**self.commitargs)
-        except Exception:
-            with self.lock:
-                writer.cancel()
-            raise
-        return count
-
-    def is_deleted(self, docnum):
-        """Check if document is deleted with temporary writer."""
-        base = self.index.refresh().doc_count_all()
-        if docnum < base:
-            writer = self.get_writer("is_deleted")
-            is_deleted = writer.is_deleted(docnum)
-            writer.cancel()
-            return is_deleted
-        return self._get_ram_reader().is_deleted(docnum - base)
-
-    def delete_by_query(self, q, searcher=None):
-        """Delete any documents matching a query object.
-
-        :returns: the number of documents deleted.
-        Called by multiprocessing.
-        """
-        s = searcher if searcher else self.searcher()
-
-        count = 0
-        try:
-            docnums = s.docs_for_query(q, for_deletion=True)
-            count = self.delete_batch_documents(docnums)
-        finally:
-            if not searcher:
-                s.close()
-        return count