codex 1.5.19__py3-none-any.whl → 1.6.0a0__py3-none-any.whl

codex/views/browser/browser_order_by.py

@@ -1,108 +1,63 @@
 """Base view for ordering the query."""
 
-from os import sep
-from types import MappingProxyType
+from django.db.models.aggregates import Max, Min
 
-from django.db.models import Avg, F, Max, Min, Sum, Value
-from django.db.models.functions import Reverse, Right, StrIndex
+from codex.models import Comic
+from codex.views.browser.filters.annotations import (
+    BrowserAnnotationsFilterView,
+)
 
-from codex.models import Comic, Folder, StoryArc
-from codex.views.browser.base import BrowserBaseView
 
-
-class BrowserOrderByView(BrowserBaseView):
+class BrowserOrderByView(BrowserAnnotationsFilterView):
     """Base class for views that need ordering."""
 
-    _ORDER_AGGREGATE_FUNCS = MappingProxyType(
-        {
-            "age_rating": Avg,
-            "community_rating": Avg,
-            "created_at": Min,
-            "critical_rating": Avg,
-            "date": Min,
-            "page_count": Sum,
-            "path": Min,
-            "size": Sum,
-            "updated_at": Min,
-            "search_score": Min,
-            "story_arc_number": Min,
-        }
-    )
-    _SEP_VALUE = Value(sep)
-    _ANNOTATED_ORDER_FIELDS = frozenset(
-        ("sort_name", "search_score", "bookmark_updated_at")
-    )
-    _COMIC_COVER_ORDERING = ("comic__" + field for field in Comic.ORDERING)
+    def __init__(self, *args, **kwargs):
+        """Set params for the type checker."""
+        super().__init__(*args, **kwargs)
+        self.order_key: str = ""
 
     def set_order_key(self):
         """Get the default order key for the view."""
-        order_key = self.params.get("order_by")
+        order_key: str = self.params.get("order_by", "")  # type: ignore
         if not order_key:
             group = self.kwargs.get("group")
-            order_key = "path" if group == self.FOLDER_GROUP else "sort_name"
-        self.order_key = order_key
-
-    @classmethod
-    def _get_path_query_func(cls, field):
-        """Use the db to get only the filename."""
-        return Right(
-            field,
-            StrIndex(Reverse(field), cls._SEP_VALUE) - 1,  # type: ignore
-        )
-
-    def get_aggregate_func(self, model, field):
-        """Order by aggregate."""
-        # get agg_func
-        agg_func = self._ORDER_AGGREGATE_FUNCS[field]
-        if agg_func == Min and self.params.get("order_reverse"):
-            agg_func = Max
-
-        # get full_field
-        self.kwargs.get("group")
-        if model == StoryArc and field == "story_arc_number":
-            full_field = "storyarcnumber__number"
-        else:
-            if self.order_key == "story_arc_number":
-                field = "story_arc_numbers__number"
-            full_field = self.rel_prefix + field
-        if field == "path":
-            full_field = self._get_path_query_func(full_field)
-
-        return agg_func(full_field)
+            order_key = "filename" if group == self.FOLDER_GROUP else "sort_name"
+        self.order_key: str = order_key
+        order_reverse = self.params.get("order_reverse")
+        self.order_agg_func = Max if order_reverse else Min
 
-    def get_order_value(self, model):
-        """Get a complete function for aggregating an attribute."""
-        # Determine order func
-        if self.order_key == "path" and model in (Comic, Folder):
-            # special path sorting.
-            func = self._get_path_query_func(self.order_key)
-        elif model == Comic or self.order_key in self._ANNOTATED_ORDER_FIELDS:
-            # agg_none uses group fields not comic fields.
-            func = F(self.order_key)  # type: ignore
-        else:
-            func = self.get_aggregate_func(model, self.order_key)
-        return func
-
-    def add_order_by(self, queryset, model, cover=False):
+    def add_order_by(self, qs, model):
         """Create the order_by list."""
-        prefix = ""
-        if self.params.get("order_reverse"):
-            prefix += "-"
-
-        if self.order_key == "sort_name":
-            if cover and model != Comic:
-                order_fields = self._COMIC_COVER_ORDERING
+        order_fields_head = None
+        if model == Comic:
+            if self.order_key == "sort_name":
+                order_fields_head = [
+                    *self.comic_sort_names,  # type: ignore
+                    "issue_number",
+                    "issue_suffix",
+                    "sort_name",
+                ]
             else:
-                order_fields = model.ORDERING
-        elif self.order_key == "bookmark_updated_at":
-            order_fields = ("order_value", "updated_at", "created_at", "pk")
-        elif self.order_key == "story_arc_number" and model == Comic:
-            order_fields = ("order_value", "date", *model.ORDERING)
+                # Comic orders on indexed fields directly
+                # Which is allegedly faster than using tmp b-trees (annotations)
+                # And since that's every cover sort, it's worth it.
+                order_fields_head = [self.order_key]
+                # Comic order micro optimizations
+                if self.order_key == "story_arc_number":
+                    order_fields_head += ["date"]
+                elif self.order_key == "bookmark_updated_at":
+                    order_fields_head += ["updated_at"]
         else:
-            order_fields = ("order_value", *model.ORDERING)
+            order_fields_head = ["order_value"]
 
+        order_fields = (*order_fields_head, "pk")
+
+        prefix = "-" if self.params.get("order_reverse") else ""
         order_by = []
-        for field in order_fields:
-            order_by.append(prefix + field)
+        if prefix:
+            for field in order_fields:
+                order_by.append(prefix + field)
+        else:
+            order_by = order_fields
 
-        return queryset.order_by(*order_by)
+        return qs.order_by(*order_by)

codex/views/browser/choices.py

@@ -11,7 +11,6 @@ from rest_framework.response import Response
 
 from codex.logger.logging import get_logger
 from codex.models import (
-    BrowserGroupModel,
     Comic,
     ContributorPerson,
     Folder,
@@ -29,18 +28,23 @@ from codex.serializers.browser.filters import (
 )
 from codex.serializers.models.pycountry import PyCountrySerializer
 from codex.views.auth import IsAuthenticatedOrEnabledNonUsers
-from codex.views.browser.base import BrowserBaseView
+from codex.views.browser.filters.annotations import (
+    BrowserAnnotationsFilterView,
+)
 
 LOG = get_logger(__name__)
 
 _FIELD_TO_REL_MODEL_MAP = MappingProxyType(
     {
-        BrowserBaseView.CONTRIBUTOR_PERSON_UI_FIELD: (
+        BrowserAnnotationsFilterView.CONTRIBUTOR_PERSON_UI_FIELD: (
             "contributors__person",
             ContributorPerson,
         ),
-        BrowserBaseView.STORY_ARC_UI_FIELD: ("story_arc_numbers__story_arc", StoryArc),
-        BrowserBaseView.IDENTIFIER_TYPE_UI_FIELD: (
+        BrowserAnnotationsFilterView.STORY_ARC_UI_FIELD: (
+            "story_arc_numbers__story_arc",
+            StoryArc,
+        ),
+        BrowserAnnotationsFilterView.IDENTIFIER_TYPE_UI_FIELD: (
             "identifiers__identifier_type",
             IdentifierType,
         ),
@@ -48,7 +52,7 @@ _FIELD_TO_REL_MODEL_MAP = MappingProxyType(
 )
 
 
-class BrowserChoicesViewBase(BrowserBaseView):
+class BrowserChoicesViewBase(BrowserAnnotationsFilterView):
     """Get choices for filter dialog."""
 
     permission_classes: ClassVar[list] = [IsAuthenticatedOrEnabledNonUsers]  # type: ignore
@@ -76,6 +80,7 @@ class BrowserChoicesViewBase(BrowserBaseView):
             IdentifierType: "identifiers__identifier_type",
         }
     )
+    _SERIALIZER_MANY = False
 
     @staticmethod
     def get_field_choices_query(comic_qs, field_name):
@@ -123,18 +128,17 @@ class BrowserChoicesViewBase(BrowserBaseView):
 
     def get_object(self):
         """Get the comic subquery use for the choices."""
-        search_scores = self.get_search_scores()
-        if search_scores is None:
-            return self.model.objects.none()  # type: ignore
-        object_filter = self.get_query_filters(self.model, search_scores, True)
-        return self.model.objects.filter(object_filter)  # type: ignore
+        object_filter = self.get_query_filters(self.model, choices=True)
+        qs = self.model.objects.filter(object_filter)  # type: ignore
+        return self.filter_by_annotations(qs, self.model, binary=True)
 
-    def _set_model(self):
-        """Set the model to query."""
-        group = self.kwargs["group"]
-        if group == self.ROOT_GROUP:
-            group = self.params.get("top_group", "p")
-        self.model: BrowserGroupModel | Comic | None = self.GROUP_MODEL_MAP[group]  # type: ignore
+    @extend_schema(request=BrowserAnnotationsFilterView.input_serializer_class)
+    def get(self, *_args, **_kwargs):
+        """Return choices."""
+        self.init_request()
+        obj = self.get_object()
+        serializer = self.get_serializer(obj, many=self._SERIALIZER_MANY)
+        return Response(serializer.data)
 
 
 class BrowserChoicesAvailableView(BrowserChoicesViewBase):
@@ -145,11 +149,14 @@ class BrowserChoicesAvailableView(BrowserChoicesViewBase):
     @classmethod
     def _get_field_choices_count(cls, comic_qs, field_name):
         """Create a pk:name object for fields without tables."""
-        return cls.get_field_choices_query(comic_qs, field_name).count()
+        qs = cls.get_field_choices_query(comic_qs, field_name)
+        return qs.count()
 
     def _get_m2m_field_choices_count(self, model, comic_qs, rel):
         """Get choices with nulls where there are nulls."""
-        count = self.get_m2m_field_query(model, comic_qs).count()
+        m2m_qs = self.get_m2m_field_query(model, comic_qs)
+        m2m_qs = m2m_qs.values("pk")
+        count = m2m_qs.count()
 
         # Detect if there are null choices.
         # Regretabbly with another query, but doing a forward query
@@ -160,14 +167,9 @@ class BrowserChoicesAvailableView(BrowserChoicesViewBase):
 
         return count
 
-    @extend_schema(request=BrowserBaseView.input_serializer_class)
-    def get(self, *_args, **_kwargs):
-        """Return all choices with more than one choice."""
-        self.parse_params()
-        self._set_model()
-        self.set_rel_prefix(self.model)
-        comic_qs = self.get_object()
-
+    def get_object(self):
+        """Get choice counts."""
+        qs = super().get_object()
         data = {}
         for field_name in self.serializer_class().get_fields():  # type: ignore
             if field_name == "story_arcs" and self.model == StoryArc:
@@ -176,9 +178,9 @@ class BrowserChoicesAvailableView(BrowserChoicesViewBase):
             rel, m2m_model = self.get_rel_and_model(field_name)
 
             if m2m_model:
-                count = self._get_m2m_field_choices_count(m2m_model, comic_qs, rel)
+                count = self._get_m2m_field_choices_count(m2m_model, qs, rel)
             else:
-                count = self._get_field_choices_count(comic_qs, rel)
+                count = self._get_field_choices_count(qs, rel)
 
             filters = self.params.get("filters", {})
             try:
@@ -186,15 +188,14 @@ class BrowserChoicesAvailableView(BrowserChoicesViewBase):
             except TypeError:
                 flag = False
             data[field_name] = flag
-
-        serializer = self.get_serializer(data)
-        return Response(serializer.data)
+        return data
 
 
 class BrowserChoicesView(BrowserChoicesViewBase):
     """Get choices for filter dialog."""
 
     # serializer_class = Dynamic class determined in get()
+    _SERIALIZER_MANY = True
 
     def _get_field_choices(self, comic_qs, field_name):
         """Create a pk:name object for fields without tables."""
@@ -238,21 +239,14 @@ class BrowserChoicesView(BrowserChoicesViewBase):
         field_name = self._get_field_name()
         return CHOICES_SERIALIZER_CLASS_MAP.get(field_name, CharListField)
 
-    @extend_schema(request=BrowserBaseView.input_serializer_class)
-    def get(self, *_args, **_kwargs):
-        """Return all choices with more than one choice."""
-        self.parse_params()
-        self._set_model()
-        self.set_rel_prefix(self.model)
-
+    def get_object(self):
+        """Return choices with more than one choice."""
+        qs = super().get_object()
         field_name = self._get_field_name()
         rel, m2m_model = self.get_rel_and_model(field_name)
 
-        comic_qs = self.get_object()
         if m2m_model:
-            choices = self._get_m2m_field_choices(m2m_model, comic_qs, rel)
+            choices = self._get_m2m_field_choices(m2m_model, qs, rel)
         else:
-            choices = self._get_field_choices(comic_qs, rel)
-
-        serializer = self.get_serializer(choices, many=True)
-        return Response(serializer.data)
+            choices = self._get_field_choices(qs, rel)
+        return choices

codex/views/browser/filters/annotations.py

@@ -0,0 +1,34 @@
+"""Annotations used by a filter."""
+
+from django.db.models.aggregates import Count
+from django.db.models.expressions import Value
+from django.db.models.fields import PositiveSmallIntegerField
+
+from codex.models.comic import Comic
+from codex.views.browser.base import BrowserBaseView
+
+
+class BrowserAnnotationsFilterView(BrowserBaseView):
+    """Annotations that also filter."""
+
+    ONE_INTEGERFIELD = Value(1, PositiveSmallIntegerField())
+
+    def _annotate_child_count(self, qs, model):
+        """Annotate Child Count."""
+        if model == Comic:
+            child_count_sum = self.ONE_INTEGERFIELD
+        else:
+            child_count_sum = Count(self.rel_prefix + "pk", distinct=True)
+        if self.TARGET == "opds2":
+            if model != Comic:
+                qs = qs.alias(child_count=child_count_sum)
+        else:
+            qs = qs.annotate(child_count=child_count_sum)
+        if model != Comic:
+            qs = qs.filter(child_count__gt=0)
+        return qs
+
+    def filter_by_annotations(self, qs, model, binary=False):
+        """Filter queryset by annotations."""
+        qs = self._annotate_child_count(qs, model)
+        return self.apply_search_filter(qs, model, binary=binary)

codex/views/browser/filters/group.py

@@ -10,18 +10,19 @@ class GroupFilterMixin(GroupACLMixin):
 
     def get_group_filter(self, choices):
         """Get filter for the displayed group."""
-        pk = self.kwargs.get("pk")  # type: ignore
-        group = self.kwargs.get("group")  # type: ignore
-        if pk:
+        group = self.kwargs["group"]  # type: ignore
+        pks = self.kwargs["pks"]  # type: ignore
+        if pks:  # type: ignore
             group_relation = "comic__" if choices else ""
             if choices and group == self.FOLDER_GROUP:
                 group_relation += "folders"
             else:
                 group_relation += self.GROUP_RELATION[group]
-            group_filter = Q(**{group_relation: pk})
+            group_relation += "__in"
+            group_filter_dict = {group_relation: pks}  # type: ignore
         elif group == self.FOLDER_GROUP:
-            group_filter = Q(parent_folder=None)
+            group_filter_dict = {"parent_folder": None}
         else:
-            group_filter = Q()
+            group_filter_dict = {}
 
-        return group_filter
+        return Q(**group_filter_dict)

codex/views/browser/filters/search.py

@@ -1,52 +1,185 @@
 """Search Filters Methods."""
 
-from django.db.models import Q
+from dataclasses import dataclass
+from time import time
+
+from django.db.models import Case, Value, When
 
-from codex._vendor.haystack.query import SearchQuerySet
 from codex.logger.logging import get_logger
 from codex.models import Comic
+from codex.search.query import CodexSearchQuerySet
+from codex.settings.settings import DEBUG
+from codex.views.browser.const import MAX_OBJ_PER_PAGE
 
 LOG = get_logger(__name__)
+SEARCH_SCORE_MAX = 100.0
+SEARCH_SCORE_MIN = 0.001
+SEARCH_SCORE_EMPTY = 0.0
+
+
+@dataclass
+class SearchScorePks:
+    """Search Scores for annotation."""
+
+    scored_pks: tuple[int, ...] = ()
+    prev_pks: tuple[int, ...] = ()
+    next_pks: tuple[int, ...] = ()
 
 
 class SearchFilterMixin:
     """Search Filters Methods."""
 
-    def get_search_scores(self) -> dict | None:
+    def _is_search_results_limited(self) -> bool:
+        """Get search result limit from params."""
+        # user = self.request.user  # type: ignore
+        # if user and user.is_authenticated:
+        #    limited = bool(
+        #        self.params.get(  # type: ignore
+        #            "search_results_limit",
+        #            MAX_OBJ_PER_PAGE,
+        #        )
+        #    )
+        # else:
+        #    limited = True
+        # return limited
+        return True
+
+    def _get_binary_search_scores(self, sqs):
+        """Get search scores for choices and metadata."""
+        sqs = sqs.values_list("pk", flat=True)
+        if self._is_search_results_limited():
+            page = self.kwargs.get("page", 1)  # type: ignore
+            limit = page * MAX_OBJ_PER_PAGE + 1
+            sqs = sqs[:limit]
+        return tuple(sqs)
+
+    def _get_browser_search_scores_params(self, is_search_results_limited):
+        page = self.kwargs.get("page", 1)  # type: ignore
+        if is_search_results_limited:
+            offset = 0
+            limit = page * MAX_OBJ_PER_PAGE + 1
+        else:
+            # For unlimited search actually take a slice
+            offset = max(0, (page - 1) * MAX_OBJ_PER_PAGE)  # type: ignore
+            limit = offset + MAX_OBJ_PER_PAGE  # type: ignore
+
+        if DEBUG:
+            LOG.debug(
+                f"Searching... {page=} {offset=} {limit=} {is_search_results_limited=}"
+            )
+        return offset, limit
+
+    def _get_browser_search_scores(self, sqs):
+        """Get search scores for the browser cards."""
+        is_search_results_limited = self._is_search_results_limited()
+        offset, limit = self._get_browser_search_scores_params(
+            is_search_results_limited
+        )
+
+        scores_pairs = []
+        prev_pks = []
+        next_pks = []
+        scored_pks = []
+
+        if DEBUG:
+            start_time = time()
+
+        order_reverse = self.params.get("order_reverse", False)  # type: ignore
+        if not order_reverse:
+            sqs = sqs.order_reverse()
+
+        scores_values = sqs.values_list("pk", "score")
+        if is_search_results_limited:
+            scores_values = scores_values[:limit]
+        for index, pair in enumerate(scores_values):
+            pk = pair[0]
+            if index < offset:
+                prev_pks.append(pk)
+            elif index > limit:
+                next_pks.append(pk)
+            else:
+                scores_pairs.append(pair)
+            scored_pks.append(pk)
+
+        if not order_reverse:
+            tmp = prev_pks
+            prev_pks = next_pks
+            next_pks = tmp
+
+        if DEBUG:
+            LOG.debug(
+                f"search {time() - start_time}s"  # type: ignore
+                f" {len(scores_pairs)=} {len(prev_pks)=} {len(next_pks)=}"
+            )
+
+        return tuple(scores_pairs), tuple(prev_pks), tuple(next_pks), tuple(scored_pks)
+
+    def _get_search_scores(self, binary=False):
         """Perform the search and return the scores as a dict."""
-        search_scores = {}
         text = self.params.get("q", "")  # type: ignore
         if not text:
-            return search_scores
+            return (), (), (), ()
 
-        sqs = SearchQuerySet().auto_query(text)
-        comic_scores = sqs.values("pk", "score")
+        score_pairs = ()
+        scored_pks = ()
+        prev_pks = ()
+        next_pks = ()
         try:
-            for comic_score in comic_scores:
-                search_scores[comic_score["pk"]] = comic_score["score"]
+            sqs = CodexSearchQuerySet()
+            sqs = sqs.auto_query(text)  # .filter(score__gt=0.0)
+            binary = binary or self.params.get("order_by") != "search_score"  # type: ignore
+            if binary:
+                scored_pks = self._get_binary_search_scores(sqs)
+            else:
+                score_pairs, prev_pks, next_pks, scored_pks = (
+                    self._get_browser_search_scores(sqs)
+                )
         except MemoryError:
             LOG.warning("Search engine needs more memory, results truncated.")
         except Exception:
-            LOG.exception("While Searching")
-        if not search_scores:
-            return None
-        return search_scores
+            LOG.exception("Getting Search Scores")
+        return score_pairs, prev_pks, next_pks, scored_pks
+
+    def annotate_search_score(  # noqa: PLR0913
+        self, qs, model, score_pairs, prev_pks=None, next_pks=None
+    ):
+        """Annotate the search score for ordering by search score.
 
-    def _get_search_query_filter(self, model, search_scores: dict):
+        Choose the maximum matching score for the group.
+        """
+        if score_pairs:
+            prefix = "" if model == Comic else self.rel_prefix  # type: ignore
+            whens = []
+            for pk, score in score_pairs:
+                when = {prefix + "pk": pk, "then": score}
+                whens.append(When(**when))
+            if prev_pks:
+                whens.append(When(pk__in=prev_pks, then=SEARCH_SCORE_MAX))
+            if next_pks:
+                whens.append(When(pk__in=next_pks, then=SEARCH_SCORE_MIN))
+            search_score = self.order_agg_func(Case(*whens, default=SEARCH_SCORE_EMPTY))  # type: ignore
+        else:
+            search_score = Value(0.0)
+        return qs.annotate(search_score=search_score)
+
+    def _get_search_query_filter(self, model, scored_pks):
         """Get the search filter and scores."""
         prefix = "" if model == Comic else self.rel_prefix  # type: ignore
         rel = prefix + "pk__in"
-        query_dict = {rel: search_scores.keys()}
-        return Q(**query_dict)
+        return {rel: scored_pks}
 
-    def get_search_filter(self, model, search_scores: dict):
+    def apply_search_filter(self, qs, model, binary=False):
         """Preparse search, search and return the filter and scores."""
-        search_filter = Q()
         try:
-            if search_scores:
-                # Query haystack
-                search_filter = self._get_search_query_filter(model, search_scores)
-        except Exception as exc:
-            LOG.warning(exc)
+            score_pairs, prev_pks, next_pks, scored_pks = self._get_search_scores(
+                binary
+            )
+            qs = self.annotate_search_score(qs, model, score_pairs, prev_pks, next_pks)
+            if score_pairs:
+                search_filter = self._get_search_query_filter(model, scored_pks)
+                qs = qs.filter(**search_filter)
+                self.cover_search_score_pairs = score_pairs
+        except Exception:
+            LOG.exception("Creating the search filter")
 
-        return search_filter
+        return qs