codex 1.6.19__py3-none-any.whl → 1.7.0__py3-none-any.whl

codex/views/browser/metadata/annotate.py

@@ -0,0 +1,109 @@
+"""Metadata Annotations."""
+
+from django.db.models import Count, IntegerField, Sum, Value
+
+from codex.logger.logging import get_logger
+from codex.models import Comic
+from codex.views.browser.annotate.card import BrowserAnnotateCardView
+
+LOG = get_logger(__name__)
+_ADMIN_OR_FILE_VIEW_ENABLED_COMIC_VALUE_FIELDS = frozenset({"path"})
+_COMIC_VALUE_FIELDS_CONFLICTING = frozenset(
+    {
+        "created_at",
+        "name",
+        "path",
+        "updated_at",
+    }
+)
+_DISABLED_VALUE_FIELD_NAMES = frozenset(
+    {"id", "pk", "sort_name", "stat"} | _COMIC_VALUE_FIELDS_CONFLICTING
+)
+_COMIC_VALUE_FIELD_NAMES = frozenset(
+    # contains path
+    field.name
+    for field in Comic._meta.get_fields()
+    if not field.is_relation and field.name not in _DISABLED_VALUE_FIELD_NAMES
+)
+_COMIC_VALUE_FIELDS_CONFLICTING_PREFIX = "conflict_"
+_COMIC_RELATED_VALUE_FIELDS = frozenset({"series__volume_count", "volume__issue_count"})
+_PATH_GROUPS = frozenset({"c", "f"})
+_SUM_FIELDS = frozenset({"page_count", "size"})
+
+
+class MetadataAnnotateView(BrowserAnnotateCardView):
+    """Metadata Annotations."""
+
+    def _get_comic_value_fields(self):
+        """Include the path field for staff."""
+        fields = set(_COMIC_VALUE_FIELD_NAMES)
+        group = self.kwargs["group"]
+        if (
+            not (self.is_admin and self.admin_flags["folder_view"])
+            or group not in _PATH_GROUPS
+        ):
+            fields -= _ADMIN_OR_FILE_VIEW_ENABLED_COMIC_VALUE_FIELDS
+        return tuple(fields)
+
+    def _intersection_annotate(
+        self,
+        querysets,
+        fields,
+        related_suffix="",
+        annotation_prefix="",
+    ):
+        """Annotate the intersection of value and fk fields."""
+        (filtered_qs, qs) = querysets
+
+        for field in fields:
+            # Annotate variant counts
+            # If 1 variant, annotate value, otherwise None
+            ann_field = annotation_prefix + field.replace("__", "_")
+
+            full_field = self.rel_prefix + field
+
+            if field in _SUM_FIELDS:
+                val = Sum(full_field)
+            else:
+                # group_by makes the filter work, but prevents its
+                # use as a subquery in the big query.
+                sq = filtered_qs.alias(
+                    filter_count=Count(full_field, distinct=True)
+                ).filter(filter_count=1)
+                sq = self.add_group_by(sq)
+                sq = self.force_inner_joins(sq)
+                sq = sq.values_list(full_field + related_suffix, flat=True)
+                try:
+                    val = sq[0]
+                except IndexError:
+                    val = None
+                if field.endswith("count"):
+                    output_field = IntegerField()
+                else:
+                    output_field = Comic._meta.get_field(field).__class__()
+                val = Value(val, output_field)
+            qs = qs.annotate(**{ann_field: val})
+
+        return filtered_qs, qs
+
+    def _annotate_values_and_fks(self, qs, filtered_qs):
+        """Annotate comic values and comic foreign key values."""
+        # Simple Values
+        querysets = (filtered_qs, qs)
+        if qs.model is not Comic:
+            comic_value_fields = self._get_comic_value_fields()
+            querysets = self._intersection_annotate(querysets, comic_value_fields)
+
+            # Conflicting Simple Values
+            querysets = self._intersection_annotate(
+                querysets,
+                _COMIC_VALUE_FIELDS_CONFLICTING,
+                annotation_prefix=_COMIC_VALUE_FIELDS_CONFLICTING_PREFIX,
+            )
+
+        # Foreign Keys with special count values
+        _, qs = self._intersection_annotate(
+            querysets,
+            _COMIC_RELATED_VALUE_FIELDS,
+        )
+        return qs

codex/views/browser/metadata/copy_intersections.py

@@ -0,0 +1,90 @@
+"""Copy Intersections Into Comic Fields."""
+
+from codex.logger.logging import get_logger
+from codex.models.comic import Comic
+from codex.serializers.browser.metadata import PREFETCH_PREFIX
+from codex.views.browser.metadata.query_intersections import (
+    MetadataQueryIntersectionsView,
+)
+
+LOG = get_logger(__name__)
+_COMIC_VALUE_FIELDS_CONFLICTING = frozenset(
+    {
+        "created_at",
+        "name",
+        "path",
+        "updated_at",
+    }
+)
+_COMIC_VALUE_FIELDS_CONFLICTING_PREFIX = "conflict_"
+_PATH_GROUPS = frozenset({"c", "f"})
+
+
+class MetadataCopyIntersectionsView(MetadataQueryIntersectionsView):
+    """Copy Intersections Into Comic Fields."""
+
+    def _path_security(self, obj):
+        """Secure filesystem information for acl situation."""
+        group = self.kwargs["group"]
+        is_path_group = group in _PATH_GROUPS
+        if is_path_group:
+            if self.is_admin:
+                return
+            if self.admin_flags["folder_view"]:
+                obj.path = obj.search_path()
+        else:
+            obj.path = ""
+
+    def _highlight_current_group(self, obj):
+        """Values for highlighting the current group."""
+        if self.model and self.model is not Comic:
+            # move the name of the group to the correct field
+            field = self.model.__name__.lower() + "_list"
+            group_list = self.model.objects.filter(pk__in=obj.ids).values("name")
+            setattr(obj, field, group_list)
+            obj.name = None
+
+    @classmethod
+    def _copy_m2m_intersections(cls, obj, m2m_intersections):
+        """Copy the m2m intersections into the object."""
+        # XXX It might even be faster to copy everything to a dict and not use the obj.
+        for key, qs in m2m_intersections.items():
+            serializer_key = (
+                f"{PREFETCH_PREFIX}{key}"
+                if key in ("identifiers", "contributors", "story_arc_numbers")
+                else key
+            )
+            if hasattr(obj, serializer_key):
+                # real db fields need to use their special set method.
+                field = getattr(obj, serializer_key)
+                field.set(
+                    qs,
+                    clear=True,
+                )
+            else:
+                # fake db field is just a queryset attached.
+                setattr(obj, serializer_key, qs)
+
+    @staticmethod
+    def _copy_groups(obj, groups):
+        for field, group_qs in groups.items():
+            setattr(obj, field + "_list", group_qs)
+
+    @staticmethod
+    def _copy_conflicting_simple_fields(obj):
+        for field in _COMIC_VALUE_FIELDS_CONFLICTING:
+            """Copy conflicting fields over naturral fields."""
+            conflict_field = _COMIC_VALUE_FIELDS_CONFLICTING_PREFIX + field
+            val = getattr(obj, conflict_field)
+            setattr(obj, field, val)
+
+    def copy_intersections_into_comic_fields(self, obj, groups, m2m_intersections):
+        """Copy a bunch of values that i couldn't fit cleanly in the main queryset."""
+        self._path_security(obj)
+        self._highlight_current_group(obj)
+        self._copy_groups(obj, groups)
+        self._copy_m2m_intersections(obj, m2m_intersections)
+        if self.model is not Comic:
+            self._copy_conflicting_simple_fields(obj)
+
+        return obj

codex/views/browser/metadata/metadata.py

@@ -0,0 +1,83 @@
+"""Aggregate Group and Comic Metadata View."""
+
+from types import MappingProxyType
+
+from drf_spectacular.utils import extend_schema
+from rest_framework.exceptions import NotFound
+from rest_framework.response import Response
+
+from codex.logger.logging import get_logger
+from codex.models import AdminFlag
+from codex.serializers.browser.metadata import MetadataSerializer
+from codex.serializers.browser.settings import BrowserFilterChoicesInputSerilalizer
+from codex.views.browser.metadata.copy_intersections import (
+    MetadataCopyIntersectionsView,
+)
+
+LOG = get_logger(__name__)
+
+
+class MetadataView(MetadataCopyIntersectionsView):
+    """Aggregate Group and Comic Metadata View."""
+
+    serializer_class = MetadataSerializer
+    input_serializer_class = BrowserFilterChoicesInputSerilalizer
+    TARGET = "metadata"
+    ADMIN_FLAG_VALUE_KEY_MAP = MappingProxyType(
+        {
+            AdminFlag.FlagChoices.FOLDER_VIEW.value: "folder_view",
+        }
+    )
+
+    def _get_valid_browse_nav_groups(self, valid_top_groups):  # noqa: ARG002
+        """Limited allowed nav groups for metadata."""
+        # Overrides method in browser.validate
+        group = self.kwargs["group"]
+        return (group,)
+
+    def _raise_not_found(self, exc=None):
+        """Raise an exception if the object is not found."""
+        pks = self.kwargs["pks"]
+        group = self.kwargs["group"]
+        detail = f"Filtered metadata for {group}/{pks} not found"
+        raise NotFound(detail=detail) from exc
+
+    def get_object(self):
+        """Create a comic-like object from the current browser group."""
+        # Comic model goes through the same code path as groups because
+        # values dicts don't copy relations to the serializer. The values
+        # dict is necessary because of the folders view union in browser.py.
+
+        qs = self.get_filtered_queryset(self.model)
+        filtered_qs = qs
+
+        # Annotate
+        qs = self.annotate_order_aggregates(qs)
+        qs = self.annotate_card_aggregates(qs)
+        qs = self._annotate_values_and_fks(qs, filtered_qs)
+        qs = self.add_group_by(qs)
+        qs = self.force_inner_joins(qs)
+
+        # Get Object
+        try:
+            obj = qs[0]
+            if not obj:
+                reason = "Empty obj"
+                raise ValueError(reason)  # noqa TRY301
+        except (IndexError, ValueError) as exc:
+            return self._raise_not_found(exc)
+
+        # Hacks to add to object after query
+        groups, m2m_intersections = self.query_intersections(filtered_qs)
+        return self.copy_intersections_into_comic_fields(obj, groups, m2m_intersections)
+
+    @extend_schema(parameters=[input_serializer_class])
+    def get(self, *_args, **_kwargs):
+        """Get metadata for a filtered browse group."""
+        # Init
+        try:
+            obj = self.get_object()
+            serializer = self.get_serializer(obj)
+            return Response(serializer.data)
+        except Exception:
+            LOG.exception(f"Getting metadata {self.kwargs}")

codex/views/browser/metadata/query_intersections.py

@@ -0,0 +1,112 @@
+"""Metadata query fk & m2m intersections."""
+
+from types import MappingProxyType
+
+from django.db.models import Count
+
+from codex.librarian.importer.const import COMIC_M2M_FIELD_NAMES
+from codex.logger.logging import get_logger
+from codex.models import Comic
+from codex.models.functions import JsonGroupArray
+from codex.views.browser.metadata.annotate import MetadataAnnotateView
+from codex.views.const import METADATA_GROUP_RELATION
+
+LOG = get_logger(__name__)
+_CONTRIBUTOR_RELATIONS = ("role", "person")
+_GROUP_RELS = MappingProxyType(
+    {
+        "i": ("publisher",),
+        "s": (
+            "publisher",
+            "imprint",
+        ),
+        "v": (
+            "publisher",
+            "imprint",
+            "series",
+        ),
+        "c": ("publisher", "imprint", "series", "volume"),
+    }
+)
+
+
+class MetadataQueryIntersectionsView(MetadataAnnotateView):
+    """Metadata query fk & m2m intersections."""
+
+    def _query_groups(self):
+        """Query the through models to show group lists."""
+        groups = {}
+        if not self.model:
+            return groups
+        group = self.kwargs["group"]
+        rel = METADATA_GROUP_RELATION.get(group)
+        if not rel:
+            return groups
+        rel = rel + "__in"
+        pks = self.kwargs["pks"]
+        group_filter = {rel: pks}
+
+        for field_name in _GROUP_RELS.get(group, ()):
+            field = self.model._meta.get_field(field_name)
+            model = field.related_model
+            if not model:
+                continue
+
+            qs = model.objects.filter(**group_filter)
+            qs = qs.only("name").distinct()
+            qs = qs.group_by("name")
+            qs = qs.annotate(ids=JsonGroupArray("id", distinct=True))
+            qs = qs.values("ids", "name")
+            groups[field_name] = qs
+        return groups
+
+    @staticmethod
+    def _get_optimized_m2m_query(key, qs):
+        if key == "contributors":
+            qs = qs.prefetch_related(*_CONTRIBUTOR_RELATIONS)
+            qs = qs.only(*_CONTRIBUTOR_RELATIONS)
+        elif key == "story_arc_numbers":
+            qs = qs.select_related("story_arc")
+            qs = qs.only("story_arc", "number")
+        elif key == "identifiers":
+            qs = qs.select_related("identifier_type")
+            qs = qs.only("identifier_type", "nss", "url")
+        else:
+            qs = qs.only("name")
+        return qs
+
+    def _get_m2m_intersection_query(self, field_name, comic_pks, comic_pks_count):
+        """Get intersection query for one field."""
+        model = Comic._meta.get_field(field_name).related_model
+        if not model:
+            reason = f"No model found for comic field: {field_name}"
+            raise ValueError(reason)
+
+        intersection_qs = model.objects.filter(comic__pk__in=comic_pks)
+        intersection_qs = intersection_qs.alias(count=Count("comic")).filter(
+            count=comic_pks_count
+        )
+        return self._get_optimized_m2m_query(field_name, intersection_qs)
+
+    def _query_m2m_intersections(self, filtered_qs):
+        """Query the through models to figure out m2m intersections."""
+        # Speed ok, but still does a query per m2m model
+        m2m_intersections = {}
+        pk_field = self.rel_prefix + "pk"
+        comic_pks = filtered_qs.distinct().values_list(pk_field, flat=True)
+        # In fts mode the join doesn't work for the query.
+        # Evaluating it now is probably faster than running the filter for every m2m anyway.
+        comic_pks = set(comic_pks)
+        comic_pks_count = len(comic_pks)
+
+        for field_name in COMIC_M2M_FIELD_NAMES:
+            m2m_intersections[field_name] = self._get_m2m_intersection_query(
+                field_name, comic_pks, comic_pks_count
+            )
+        return m2m_intersections
+
+    def query_intersections(self, filtered_qs):
+        """Query complex intersections."""
+        groups = self._query_groups()
+        m2m_intersections = self._query_m2m_intersections(filtered_qs)
+        return groups, m2m_intersections

codex/views/browser/mtime.py

@@ -7,13 +7,13 @@ from codex.logger.logging import get_logger
 from codex.models.groups import Publisher
 from codex.serializers.browser.mtime import GroupsMtimeSerializer, MtimeSerializer
 from codex.util import max_none
-from codex.views.browser.filters.annotations import BrowserAnnotationsFilterView
+from codex.views.browser.group_mtime import BrowserGroupMtimeView
 from codex.views.const import GROUP_MODEL_MAP
 
 LOG = get_logger(__name__)
 
 
-class MtimeView(BrowserAnnotationsFilterView):
+class MtimeView(BrowserGroupMtimeView):
     """Get the mtimes for the submitted groups."""
 
     input_serializer_class = GroupsMtimeSerializer
@@ -22,11 +22,6 @@ class MtimeView(BrowserAnnotationsFilterView):
     REPARSE_JSON_FIELDS = frozenset({"groups", "filters"})
     TARGET = "mtime"
 
-    def __init__(self, *args, **kwargs) -> None:
-        """Initialize."""
-        super().__init__(*args, **kwargs)
-        self.init_bookmark_data()
-
     def _get_group_mtime(self, item):
         """Get one group's mtimes."""
         group = item["group"]
@@ -50,9 +45,6 @@ class MtimeView(BrowserAnnotationsFilterView):
     @extend_schema(parameters=[GroupsMtimeSerializer])
     def get(self, *args, **kwargs):
         """Get the mtimes for the submitted groups."""
-        # Parse Request
-        self.parse_params()
-
         max_mtime = self.get_max_groups_mtime()
 
         # Serialize Response

codex/views/browser/order_by.py

@@ -1,21 +1,29 @@
 """Base view for ordering the query."""
 
-from django.db.models.aggregates import Max, Min
-
 from codex.models import Comic
-from codex.views.browser.filters.annotations import (
-    BrowserAnnotationsFilterView,
-)
+from codex.views.browser.group_mtime import BrowserGroupMtimeView
 
 
-class BrowserOrderByView(BrowserAnnotationsFilterView):
+class BrowserOrderByView(BrowserGroupMtimeView):
     """Base class for views that need ordering."""
 
-    def set_order_key(self):
+    def __init__(self, *args, **kwargs):
+        """Initialize memoized vars."""
+        super().__init__(*args, **kwargs)
+        self._order_key: str = ""
+        self._comic_sort_names: tuple[str, ...] = ()
+
+    @property
+    def order_key(self):
         """Get the default order key for the view."""
-        self.order_key: str = self.params["order_by"]
-        order_reverse = self.params.get("order_reverse")
-        self.order_agg_func = Max if order_reverse else Min
+        if not self._order_key:
+            order_key: str = self.params["order_by"]
+            if (order_key == "search_score" and not self.fts_mode) or (
+                order_key == "filename" and not self.admin_flags["folder_view"]
+            ):
+                order_key = "sort_name"
+            self._order_key = order_key
+        return self._order_key
 
     def _add_comic_order_by(self, order_key, comic_sort_names):
         """Order by for comics (and covers)."""
@@ -23,7 +31,7 @@ class BrowserOrderByView(BrowserAnnotationsFilterView):
             order_key = self.order_key
         if order_key == "sort_name":
             if not comic_sort_names:
-                comic_sort_names = self.comic_sort_names  # type: ignore
+                comic_sort_names = self._comic_sort_names
             order_fields_head = [
                 *comic_sort_names,
                 "issue_number",
@@ -39,15 +47,13 @@ class BrowserOrderByView(BrowserAnnotationsFilterView):
             if order_key == "story_arc_number":
                 order_fields_head += ["date"]
             elif order_key == "bookmark_updated_at":
-                order_fields_head += ["updated_at"]
+                order_fields_head += ["bookmark_updated_at"]
         return order_fields_head
 
-    def add_order_by(
-        self, qs, model, order_key="", do_reverse=True, comic_sort_names=None
-    ):
+    def add_order_by(self, qs, order_key="", do_reverse=True, comic_sort_names=None):
         """Create the order_by list."""
         order_fields_head = ()
-        if model == Comic:
+        if qs.model is Comic:
             order_fields_head = self._add_comic_order_by(order_key, comic_sort_names)
         else:
             order_fields_head = ["order_value"]

codex/views/browser/page_in_bounds.py

@@ -0,0 +1,88 @@
+"""Browser Page Bounds Checking."""
+
+from codex.logger.logging import get_logger
+from codex.views.browser.annotate.card import BrowserAnnotateCardView
+from codex.views.util import Route
+
+LOG = get_logger(__name__)
+
+
+class BrowserPageInBoundsView(BrowserAnnotateCardView):
+    """Browser Page Bounds Checking."""
+
+    def _get_back_one_page_route(self, num_pages):
+        """Get max page if oob or 1."""
+        LOG.debug("Redirect back one page.")
+        group = self.kwargs.get("group")
+        pks = self.kwargs.get("pks")
+        page = self.kwargs.get("page", 1)
+        new_page = num_pages if num_pages and page > num_pages else 1
+        pks = pks if pks else "0"
+        return {"group": group, "pks": pks, "page": new_page}
+
+    def _get_up_breadcrumbs(self):
+        """Walk up the breadcrumbs to get the next level up."""
+        breadcrumbs = self.params.get("breadcrumbs", [])
+        new_breadcrumbs = []
+        level = False
+        group = self.kwargs.get("group")
+        pks = self.kwargs.get("pks")
+        page = self.kwargs.get("page", 1)
+        current_route = Route(group=group, pks=pks, page=page)
+        for crumb in reversed(breadcrumbs):
+            if not level:
+                crumb_route = Route(**crumb)
+                level = current_route & crumb_route
+                continue
+            new_breadcrumbs = [crumb, *new_breadcrumbs]
+
+        if not new_breadcrumbs:
+            if group not in ("f", "a"):
+                group = "r"
+            top_route = {"group": group, "pks": (), "page": 1}
+            new_breadcrumbs = [top_route]
+        return new_breadcrumbs
+
+    def _get_up_page_redirect(self):
+        """Walk up the breadcrumbs."""
+        try:
+            up_breadcrumbs = self._get_up_breadcrumbs()
+            route_mask = up_breadcrumbs[-1]
+            settings_mask = {"breadcrumbs": up_breadcrumbs}
+            LOG.debug("Redirect up a level.")
+        except IndexError:
+            group = self.kwargs.get("group")
+            pks = "0"
+            new_page = 1
+            route_mask = {"group": group, "pks": pks, "page": new_page}
+            settings_mask = {"breadcrumbs": [route_mask]}
+            LOG.debug("Redirect to all at current group.")
+        return route_mask, settings_mask
+
+    def _handle_page_out_of_bounds(self, num_pages):
+        """Handle out of bounds redirect."""
+        # Try to find a logical page to run to.
+        group = self.kwargs.get("group")
+        page = self.kwargs.get("page", 1)
+        pks = self.kwargs.get("pks")
+        reason = f"{group=} {pks=} {page=} does not exist."
+
+        # Adjust route mask for redirect
+        if num_pages and page > 1:
+            route_mask = self._get_back_one_page_route(num_pages)
+            settings_mask = None
+        else:
+            # This now only occurs when page < 1
+            route_mask, settings_mask = self._get_up_page_redirect()
+
+        self.raise_redirect(reason, route_mask=route_mask, settings_mask=settings_mask)
+
+    def check_page_in_bounds(self, num_pages: int):
+        """Redirect page out of bounds."""
+        page = self.kwargs.get("page", 1)
+        if page == 1 or (page >= 1 and page <= num_pages):
+            # Don't redirect if on the root page for the group.
+            # Or page within valid range.
+            return
+
+        self._handle_page_out_of_bounds(num_pages)