codex 1.5.19__py3-none-any.whl → 1.6.0a0__py3-none-any.whl

codex/models/comic.py

@@ -66,13 +66,10 @@ class Comic(WatchedPath):
         CBT = "CBT"
         PDF = "PDF"
 
-    ORDERING = (
-        "series__name",
-        "volume__name",
+    _ORDERING = (
         "issue_number",
         "issue_suffix",
-        "name",
-        "pk",
+        "sort_name",
     )
     _RE_COMBINE_WHITESPACE = re.compile(r"\s+")
 
@@ -199,6 +196,7 @@ class Comic(WatchedPath):
 
     def presave(self):
         """Set computed values."""
+        super().presave()
         self._set_date()
         self._set_decade()
         self.size = Path(self.path).stat().st_size
@@ -208,11 +206,6 @@ class Comic(WatchedPath):
         """Calculate max page from page_count."""
         return max(self.page_count - 1, 0)
 
-    def save(self, *args, **kwargs):
-        """Save computed fields."""
-        self.presave()
-        super().save(*args, **kwargs)
-
     @staticmethod
     def _compute_zero_pad(issue_number_max):
         """Compute zero padding for issues."""
@@ -222,9 +215,31 @@ class Comic(WatchedPath):
             return 1
         return math.floor(math.log10(issue_number_max)) + 1
 
+    def get_filename(self):
+        """Return filename from path as a property."""
+        return Path(self.path).name
+
+    @classmethod
+    def _get_title_issue_str(cls, obj, zero_pad):
+        """Get the issue parts of the title."""
+        issue_str = ""
+        if obj.issue_number is not None:
+            issue_number = obj.issue_number.normalize()
+            if not zero_pad:
+                zero_pad = 3
+            if issue_number % 1 == 0:
+                precision = 0
+            else:
+                precision = 1
+                zero_pad += 2
+            issue_str = f"#{issue_number:0{zero_pad}.{precision}f}"
+        if issue_suffix := obj.issue_suffix:
+            issue_str += issue_suffix
+        return issue_str
+
     @classmethod
     def get_title(  # noqa: PLR0913
-        cls, obj, volume=True, issue_number_max=None, name=True, filename_fallback=False
+        cls, obj, volume=True, zero_pad=None, name=True, filename_fallback=False
     ):
         """Create the comic title for display."""
         names = []
@@ -239,19 +254,7 @@ class Comic(WatchedPath):
             names.append(vn)
 
         # Issue
-        issue_str = ""
-        if obj.issue_number is not None:
-            issue_number = obj.issue_number.normalize()
-            zero_pad = cls._compute_zero_pad(issue_number_max)
-            if issue_number % 1 == 0:
-                precision = 0
-            else:
-                precision = 1
-                zero_pad += 2
-            issue_str = f"#{issue_number:0{zero_pad}.{precision}f}"
-        if issue_suffix := obj.issue_suffix:
-            issue_str += issue_suffix
-        if issue_str:
+        if issue_str := cls._get_title_issue_str(obj, zero_pad):
             names.append(issue_str)
 
         # Title
@@ -261,19 +264,9 @@ class Comic(WatchedPath):
         title = " ".join(filter(None, names)).strip(" .")
         title = cls._RE_COMBINE_WHITESPACE.sub(" ", title).strip()
         if filename_fallback and not title:
-            title = cls.get_filename(obj)
+            title = obj.get_filename()
         return title
 
-    @classmethod
-    def get_filename(cls, obj):
-        """Get the fileaname from dict."""
-        path = Path(obj.path)
-        return path.stem + path.suffix
-
-    def filename(self):
-        """Create a filename for download."""
-        return self.get_filename(self)
-
     def __str__(self):
         """Most common text representation for logging."""
         return self.get_title(self, filename_fallback=True)
@@ -286,3 +279,7 @@ class Comic(WatchedPath):
         if self.issue_suffix:
             res += self.issue_suffix
         return res
+
+    def search_path(self) -> str:
+        """Relative path for search index."""
+        return self.path.removeprefix(self.library.path)

codex/models/const.py

@@ -0,0 +1,18 @@
+"""Constants for models."""
+
+ARTICLES = frozenset(
+    ("a", "an", "the")  # en    # noqa RUF005
+    + ("un", "unos", "unas", "el", "los", "la", "las")  # es
+    + ("un", "une", "le", "les", "la", "les", "l'")  # fr
+    + ("o", "a", "os")  # pt
+    # pt "as" conflicts with English
+    + ("der", "dem", "des", "das")  # de
+    # de: "den & die conflict with English
+    + ("il", "lo", "gli", "la", "le", "l'")  # it
+    # it: "i" conflicts with English
+    + ("de", "het", "een")  # nl
+    + ("en", "ett")  # sw
+    + ("en", "ei", "et")  # no
+    + ("en", "et")  # da
+    + ("el", "la", "els", "les", "un", "una", "uns", "unes", "na")  # ct
+)

codex/models/functions.py

@@ -0,0 +1,12 @@
+"""Custom Django DB functions."""
+
+from django.db.models import Aggregate, JSONField
+
+
+class JsonGroupArray(Aggregate):
+    """Sqlite3 json_group_array() function."""
+
+    allow_distinct = True
+    function = "JSON_GROUP_ARRAY"
+    name = "JsonGroupArray"
+    output_field = JSONField()  # type: ignore

codex/models/groups.py

@@ -9,6 +9,7 @@ from django.db.models import (
 )
 
 from codex.models.base import MAX_NAME_LEN, BaseModel
+from codex.models.const import ARTICLES
 
 __all__ = ("BrowserGroupModel", "Publisher", "Imprint", "Series", "Volume")
 
@@ -17,9 +18,31 @@ class BrowserGroupModel(BaseModel):
     """Browser groups."""
 
     DEFAULT_NAME = ""
-    ORDERING = ("name", "pk")
+    PARENT = ""
 
     name = CharField(db_index=True, max_length=MAX_NAME_LEN, default=DEFAULT_NAME)
+    sort_name = CharField(db_index=True, max_length=MAX_NAME_LEN, default=DEFAULT_NAME)
+
+    def set_sort_name(self):
+        """Create sort_name for model."""
+        lower_name = self.name.lower()
+        sort_name = lower_name
+        name_parts = lower_name.split()
+        if len(name_parts) > 1:
+            first_word = name_parts[0]
+            if first_word in ARTICLES:
+                sort_name = " ".join(name_parts[1:])
+                sort_name += ", " + first_word
+        self.sort_name = sort_name
+
+    def presave(self):
+        """Set computed values."""
+        self.set_sort_name()
+
+    def save(self, *args, **kwargs):
+        """Save computed fields."""
+        self.presave()
+        super().save(*args, **kwargs)
 
     class Meta(BaseModel.Meta):
         """Without this a real table is created and joined to."""
@@ -39,7 +62,7 @@ class Publisher(BrowserGroupModel):
 class Imprint(BrowserGroupModel):
     """A Publishing imprint."""
 
-    ORDERING = ("publisher__name", "name", "pk")
+    PARENT = "publisher"
 
     publisher = ForeignKey(Publisher, on_delete=CASCADE)
 
@@ -52,6 +75,8 @@ class Imprint(BrowserGroupModel):
 class Series(BrowserGroupModel):
     """The series the comic belongs to."""
 
+    PARENT = "imprint"
+
     publisher = ForeignKey(Publisher, on_delete=CASCADE)
     imprint = ForeignKey(Imprint, on_delete=CASCADE)
     volume_count = PositiveSmallIntegerField(null=True)
@@ -67,7 +92,7 @@ class Volume(BrowserGroupModel):
     """The volume of the series the comic belongs to."""
 
     DEFAULT_NAME = None
-    ORDERING = ("series__name", "name", "pk")
+    PARENT = "series"
     YEAR_LEN = 4
 
     publisher = ForeignKey(Publisher, on_delete=CASCADE)
@@ -76,6 +101,12 @@ class Volume(BrowserGroupModel):
     issue_count = PositiveSmallIntegerField(null=True)
     name = SmallIntegerField(db_index=True, null=True, default=DEFAULT_NAME)
 
+    # Harmful because name is numeric
+    sort_name = None
+
+    def set_sort_name(self):
+        """Noop."""
+
     class Meta(BrowserGroupModel.Meta):
         """Constraints."""
 
@@ -84,11 +115,12 @@ class Volume(BrowserGroupModel):
     @classmethod
     def to_str(cls, name):
         """Represent volume as a string."""
-        if name in (None, ""):
-            return ""
-
-        name = str(name)
-        return f"({name})" if len(name) == cls.YEAR_LEN else "v" + name
+        if name is None:
+            rep = ""
+        else:
+            name = str(name)
+            rep = f"({name})" if len(name) == cls.YEAR_LEN else "v" + name
+        return rep
 
     def __str__(self):
         """Represent volume as a string."""

codex/models/paths.py

@@ -41,6 +41,11 @@ class WatchedPath(BrowserGroupModel):
         st[8] = st_record.st_mtime
         self.stat = st
 
+    def presave(self):
+        """Save stat."""
+        super().presave()
+        self.set_stat()
+
     def __str__(self):
         """Return the full path."""
         return self.path
@@ -51,10 +56,6 @@ class WatchedPath(BrowserGroupModel):
         unique_together = ("library", "path")
         abstract = True
 
-    def search_path(self) -> str:
-        """Relative path for search index."""
-        return self.path.removeprefix(self.library.path)
-
 
 class Folder(WatchedPath):
     """File system folder."""

codex/models/query.py

@@ -0,0 +1,91 @@
+"""Custom Compiler to force group_by."""
+# If any group_by() is attached to the QuerySet, it completely overrides the compiler computed group_by
+
+from django.db import connections
+from django.db.models import Manager
+from django.db.models.query import QuerySet
+from django.db.models.sql.compiler import SQLCompiler
+from django.db.models.sql.query import Query
+
+
+class GroupBySQLCompiler(SQLCompiler):
+    """Custom Compiler to force group_by."""
+
+    def __init__(self, *args, **kwargs):
+        """Initialize force group_by fields."""
+        super().__init__(*args, **kwargs)
+        self.force_group_by_table = ""
+        self.force_group_by_fields = ()
+
+    def set_force_group_by(self, table, fields):
+        """Set the force group_by variables."""
+        self.force_group_by_table = table
+        self.force_group_by_fields = fields
+
+    def get_group_by(self, *args, **kwargs):
+        """If force group_by set, force it."""
+        if self.force_group_by_table and self.force_group_by_fields:
+            table = self.force_group_by_table
+            group_by = []
+            for field in self.force_group_by_fields:
+                field_str = f'"{table}"."{field}"'
+                entry = (field_str, ())
+                group_by.append(entry)
+        else:
+            group_by = super().get_group_by(*args, **kwargs)
+        return group_by
+
+
+class GroupByQuery(Query):
+    """Custom Query to use GroupBy Compiler."""
+
+    def __init__(self, *args, **kwargs):
+        """Init force group_by fields."""
+        super().__init__(*args, **kwargs)
+        self.force_group_by_table = ""
+        self.force_group_by_fields = ()
+
+    def get_compiler(self, using=None, connection=None, elide_empty=True):
+        """Use the custom compiler instead of SQLCompiler."""
+        if self.compiler == "SQLCompiler":
+            if using is None and connection is None:
+                reason = "Need either using or connection"
+                raise ValueError(reason)
+            if using:
+                connection = connections[using]
+            compiler = GroupBySQLCompiler(self, connection, using, elide_empty)
+            compiler.set_force_group_by(
+                self.force_group_by_table, self.force_group_by_fields
+            )
+        else:
+            # Normal super() operation.
+            compiler = super().get_compiler(using, connection, elide_empty)  # type: ignore
+
+        return compiler
+
+    def set_force_group_by(self, fields, model=None):
+        """Set the force group_by fields."""
+        if not model:
+            model = self.model
+        table = model._meta.db_table if model else ""
+        self.force_group_by_table = table
+        self.force_group_by_fields = fields
+
+
+class GroupByQuerySet(QuerySet):
+    """Custom Queryset that uses Custom compiler."""
+
+    def __init__(self, model=None, query=None, using=None, hints=None):
+        """Use the custom query with the custom compiler."""
+        query = query or GroupByQuery(model)
+        super().__init__(model=model, query=query, using=using, hints=hints)
+
+    def group_by(self, *fields, model=None):
+        """Force group_by operator."""
+        obj = self._chain()  # type: ignore
+        obj.query.set_force_group_by(fields, model=model)
+        return obj
+
+
+class GroupByManager(Manager.from_queryset(GroupByQuerySet)):
+    """Use GroupBy QuerySet."""

codex/search/backend.py

@@ -36,6 +36,7 @@ from codex.librarian.search.status import SearchIndexStatusTypes
 from codex.logger.logging import get_logger
 from codex.memory import get_mem_limit
 from codex.models import AdminFlag, Comic
+from codex.search.backend_search import CodexBackendSearchMixin
 from codex.search.indexes import ComicIndex
 from codex.search.writing import CodexWriter
 from codex.settings.settings import (
@@ -103,7 +104,7 @@ TEXT_ANALYZER = (
 )
 
 
-class CodexSearchBackend(WhooshSearchBackend, WorkerBaseMixin):
+class CodexSearchBackend(CodexBackendSearchMixin, WhooshSearchBackend, WorkerBaseMixin):
     """Custom Whoosh Backend."""
 
     FIELDMAP = MappingProxyType(
@@ -266,7 +267,6 @@ class CodexSearchBackend(WhooshSearchBackend, WorkerBaseMixin):
                     stored=True,
                     analyzer=TEXT_ANALYZER,
                     field_boost=field_class.boost,
-                    spelling=field_class.document is True,
                 )
                 if field_class.document:
                     content_field_name = index_fieldname

codex/search/backend_search.py

@@ -0,0 +1,232 @@
+"""Backend search that gets the reverse variable as a parameter."""
+
+import warnings
+from datetime import date, datetime
+
+from _vendor.haystack.backends import (
+    log_query,
+)
+from _vendor.haystack.constants import (
+    DJANGO_CT,
+)
+from _vendor.haystack.exceptions import SearchBackendError
+from _vendor.haystack.utils import get_model_ct
+from django.conf import settings
+from django.utils.encoding import force_str
+
+# Bubble up the correct error.
+from whoosh.sorting import Count, DateRangeFacet, FieldFacet
+from whoosh.support.relativedelta import relativedelta as RelativeDelta
+
+
+class CodexBackendSearchMixin:
+    @log_query
+    def search(
+        self,
+        query_string,
+        sort_by=None,
+        start_offset=0,
+        end_offset=None,
+        fields="",
+        highlight=False,
+        facets=None,
+        date_facets=None,
+        query_facets=None,
+        narrow_queries=None,
+        spelling_query=None,
+        within=None,
+        dwithin=None,
+        distance_point=None,
+        models=None,
+        limit_to_registered_models=None,
+        result_class=None,
+        reverse=False,
+        **kwargs,
+    ):
+        if not self.setup_complete:
+            self.setup()
+
+        # A zero length query should return no results.
+        if len(query_string) == 0:
+            return {"results": [], "hits": 0}
+
+        query_string = force_str(query_string)
+
+        # A one-character query (non-wildcard) gets nabbed by a stopwords
+        # filter and should yield zero results.
+        if len(query_string) <= 1 and query_string != "*":
+            return {"results": [], "hits": 0}
+
+        # reverse = False # old reverse flag
+
+        if sort_by is not None:
+            # Determine if we need to reverse the results and if Whoosh can
+            # handle what it's being asked to sort by. Reversing is an
+            # all-or-nothing action, unfortunately.
+            sort_by_list = []
+            reverse_counter = 0
+
+            for order_by in sort_by:
+                if order_by.startswith("-"):
+                    reverse_counter += 1
+            if reverse_counter and reverse_counter != len(sort_by):
+                raise SearchBackendError(
+                    "Whoosh requires all order_by fields"
+                    " to use the same sort direction"
+                )
+
+            for order_by in sort_by:
+                if order_by.startswith("-"):
+                    sort_by_list.append(order_by[1:])
+
+                    if len(sort_by_list) == 1:
+                        reverse = True
+                else:
+                    sort_by_list.append(order_by)
+
+                    if len(sort_by_list) == 1:
+                        reverse = False
+
+            sort_by = sort_by_list
+
+        group_by = []
+        facet_types = {}
+        if facets is not None:
+            group_by += [
+                FieldFacet(facet, allow_overlap=True, maptype=Count) for facet in facets
+            ]
+            facet_types.update({facet: "fields" for facet in facets})
+
+        if date_facets is not None:
+
+            def _fixup_datetime(dt):
+                if isinstance(dt, datetime):
+                    return dt
+                if isinstance(dt, date):
+                    return datetime(dt.year, dt.month, dt.day)
+                raise ValueError
+
+            for key, value in date_facets.items():
+                start = _fixup_datetime(value["start_date"])
+                end = _fixup_datetime(value["end_date"])
+                gap_by = value["gap_by"]
+                gap_amount = value.get("gap_amount", 1)
+                gap = RelativeDelta(**{"%ss" % gap_by: gap_amount})
+                group_by.append(DateRangeFacet(key, start, end, gap, maptype=Count))
+                facet_types[key] = "dates"
+
+        if query_facets is not None:
+            warnings.warn(
+                "Whoosh does not handle query faceting.", Warning, stacklevel=2
+            )
+
+        narrowed_results = None
+        self.index = self.index.refresh()
+
+        if limit_to_registered_models is None:
+            limit_to_registered_models = getattr(
+                settings, "HAYSTACK_LIMIT_TO_REGISTERED_MODELS", True
+            )
+
+        if models and len(models):
+            model_choices = sorted(get_model_ct(model) for model in models)
+        elif limit_to_registered_models:
+            # Using narrow queries, limit the results to only models handled
+            # with the current routers.
+            model_choices = self.build_models_list()
+        else:
+            model_choices = []
+
+        if len(model_choices) > 0:
+            if narrow_queries is None:
+                narrow_queries = set()
+
+            narrow_queries.add(
+                " OR ".join(["%s:%s" % (DJANGO_CT, rm) for rm in model_choices])
+            )
+
+        narrow_searcher = None
+
+        if narrow_queries is not None:
+            # Potentially expensive? I don't see another way to do it in Whoosh...
+            narrow_searcher = self.index.searcher()
+
+            for nq in narrow_queries:
+                recent_narrowed_results = narrow_searcher.search(
+                    self.parser.parse(force_str(nq)), limit=None
+                )
+
+                if len(recent_narrowed_results) <= 0:
+                    return {"results": [], "hits": 0}
+
+                if narrowed_results is not None:
+                    narrowed_results.filter(recent_narrowed_results)
+                else:
+                    narrowed_results = recent_narrowed_results
+
+        self.index = self.index.refresh()
+
+        if self.index.doc_count():
+            searcher = self.index.searcher()
+            parsed_query = self.parser.parse(query_string)
+
+            # In the event of an invalid/stopworded query, recover gracefully.
+            if parsed_query is None:
+                return {"results": [], "hits": 0}
+
+            page_num, page_length = self.calculate_page(start_offset, end_offset)
+
+            search_kwargs = {
+                "pagelen": page_length,
+                "sortedby": sort_by,
+                "reverse": reverse,
+                "groupedby": group_by,
+            }
+
+            # Handle the case where the results have been narrowed.
+            if narrowed_results is not None:
+                search_kwargs["filter"] = narrowed_results
+
+            try:
+                raw_page = searcher.search_page(parsed_query, page_num, **search_kwargs)
+            except ValueError:
+                if not self.silently_fail:
+                    raise
+
+                return {"results": [], "hits": 0, "spelling_suggestion": None}
+
+            # Because as of Whoosh 2.5.1, it will return the wrong page of
+            # results if you request something too high. :(
+            if raw_page.pagenum < page_num:
+                return {"results": [], "hits": 0, "spelling_suggestion": None}
+
+            results = self._process_results(
+                raw_page,
+                highlight=highlight,
+                query_string=query_string,
+                spelling_query=spelling_query,
+                result_class=result_class,
+                facet_types=facet_types,
+            )
+            searcher.close()
+
+            if hasattr(narrow_searcher, "close"):
+                narrow_searcher.close()
+
+            return results
+        else:
+            if self.include_spelling:
+                if spelling_query:
+                    spelling_suggestion = self.create_spelling_suggestion(
+                        spelling_query
+                    )
+                else:
+                    spelling_suggestion = self.create_spelling_suggestion(query_string)
+            else:
+                spelling_suggestion = None
+
+            return {
+                "results": [],
+                "hits": 0,
+                "spelling_suggestion": spelling_suggestion,
+            }

codex/search/engine.py

@@ -3,10 +3,10 @@
 from codex._vendor.haystack.backends import UnifiedIndex
 from codex._vendor.haystack.backends.whoosh_backend import (
     WhooshEngine,
-    WhooshSearchQuery,
 )
 from codex.search.backend import CodexSearchBackend
 from codex.search.indexes import ComicIndex
+from codex.search.query import CodexSearchQuery
 
 
 class CodexUnifiedIndex(UnifiedIndex):
@@ -18,14 +18,6 @@ class CodexUnifiedIndex(UnifiedIndex):
         return [ComicIndex()]
 
 
-class CodexSearchQuery(WhooshSearchQuery):
-    """Custom search qeuery."""
-
-    def clean(self, query_fragment):
-        """Optimize to noop because RESERVED_ consts are empty in the backend."""
-        return query_fragment
-
-
 class CodexSearchEngine(WhooshEngine):
     """A search engine with a locking backend."""
 

codex/search/query.py

@@ -0,0 +1,44 @@
+"""Custom Codex search query classes."""
+
+from codex._vendor.haystack.backends.whoosh_backend import (
+    WhooshSearchQuery,
+)
+from codex._vendor.haystack.query import SearchQuerySet
+
+
+class CodexSearchQuery(WhooshSearchQuery):
+    """Custom search qeuery."""
+
+    def __init__(self, *args, **kwargs):
+        """Give CodexSearchQuery a reverse attribute."""
+        super().__init__(*args, **kwargs)
+        self._reverse = False
+
+    def clean(self, query_fragment):
+        """Optimize to noop because RESERVED_ consts are empty in the backend."""
+        return query_fragment
+
+    def order_reverse(self):
+        """Invert the reverse attribute."""
+        self._reverse = not self._reverse
+
+    def get_results(self, **kwargs):
+        """Send custom get results the reverse flag."""
+        return super().get_results(reverse=self._reverse, **kwargs)
+
+
+class CodexSearchQuerySet(SearchQuerySet):
+    """SearchQuerySet with order_reverse method."""
+
+    def order_reverse(self):
+        """Alters the order in which the results should appear."""
+        clone = self._clone()
+        clone.query.order_reverse()  # type: ignore
+        return clone
+
+    def values_list(self, *args, **kwargs):
+        """Copy reverse setting into CodexSearchQuery."""
+        clone = super().values_list(*args, **kwargs)
+        if clone.query and self.query:
+            clone.query._reverse = self.query._reverse  # noqa: SLF001
+        return clone