codex 1.6.19__py3-none-any.whl → 1.7.0a0__py3-none-any.whl

codex/views/browser/filters/annotations.py

@@ -1,8 +1,12 @@
 """Annotations used by a filter."""
 
+from logging import DEBUG, WARNING
+
 from django.db.models.aggregates import Count, Max
 from django.db.models.functions import Coalesce, Greatest
+from django.db.utils import OperationalError
 
+from codex.logger.logging import get_logger
 from codex.models.comic import Comic
 from codex.views.browser.filters.bookmark import BookmarkFilterMixin
 from codex.views.browser.validate import BrowserValidateView
@@ -12,11 +16,15 @@ from codex.views.const import (
 )
 
 _CHILD_COUNT = "child_count"
+_FTS5_PREFIX = "fts5: "
+LOG = get_logger(__name__)
 
 
 class BrowserAnnotationsFilterView(BrowserValidateView, BookmarkFilterMixin):
     """Annotations that also filter."""
 
+    TARGET = ""
+
     def __init__(self, *args, **kwargs) -> None:
         """Initialize."""
         super().__init__(*args, **kwargs)
@@ -72,6 +80,16 @@ class BrowserAnnotationsFilterView(BrowserValidateView, BookmarkFilterMixin):
         qs = self.apply_search_filter(qs, model)
         return self._filter_by_child_count(qs, model)
 
+    def _handle_operational_error(self, err):
+        msg = err.args[0] if err.args else ""
+        if msg.startswith(_FTS5_PREFIX):
+            level = DEBUG
+            self.search_error = msg.removeprefix(_FTS5_PREFIX)
+        else:
+            level = WARNING
+            msg = str(err)
+        LOG.log(level, f"Query Error: {msg}")
+
     def get_group_mtime(self, model, group=None, pks=None, page_mtime=False):
         """Get a filtered mtime for browser pages and mtime checker."""
         qs = self.get_filtered_queryset(
@@ -92,12 +110,16 @@ class BrowserAnnotationsFilterView(BrowserValidateView, BookmarkFilterMixin):
             mbua = EPOCH_START_DATETIMEFIELD
         qs = qs.annotate(max_bookmark_updated_at=mbua)
 
-        mtime = qs.aggregate(
-            max=Greatest(
-                Coalesce("max_bookmark_updated_at", EPOCH_START_DATETIMEFIELD),
-                "max_updated_at",
-            )
-        )["max"]
+        try:
+            mtime = qs.aggregate(
+                max=Greatest(
+                    Coalesce("max_bookmark_updated_at", EPOCH_START_DATETIMEFIELD),
+                    "max_updated_at",
+                )
+            )["max"]
+        except OperationalError as exc:
+            self._handle_operational_error(exc)
+            mtime = None
         if mtime == NotImplemented:
             mtime = None
         return mtime

codex/views/browser/filters/search/__init__.py

@@ -0,0 +1 @@
+"""Search filter."""

codex/views/browser/filters/search/aliases.py

@@ -0,0 +1,81 @@
+"""Field aliases and types maps."""
+
+from types import MappingProxyType
+
+from django.db.models.fields import CharField
+from django.db.models.fields.related import ManyToManyField
+
+from codex.models.comic import Comic
+
+
+def gen_multipart_field_aliases(field):
+    """Generate aliases for fields made of snake_case words."""
+    bits = field.split("_")
+    aliases = []
+
+    # Singular from plural
+    if field.endswith("s"):
+        aliases += [field[:-1]]
+
+    # Alternate delimiters
+    for connector in ("", "-"):
+        joined = connector.join(bits)
+        aliases += [joined, joined[:-1]]
+
+    return tuple(sorted(frozenset(aliases)))
+
+
+FIELDMAP = MappingProxyType(
+    {
+        "characters": ("category", "character"),
+        "community_rating": gen_multipart_field_aliases("community_rating"),
+        "contributors": (
+            *gen_multipart_field_aliases("authors"),
+            "contributor",
+            *gen_multipart_field_aliases("creators"),
+            *gen_multipart_field_aliases("credits"),
+            *gen_multipart_field_aliases("people"),
+            *gen_multipart_field_aliases("persons"),
+        ),
+        "created_at": ("created",),
+        "critical_rating": gen_multipart_field_aliases("critical_rating"),
+        "genres": ("genre",),
+        "file_type": (
+            "filetype",
+            "type",
+        ),
+        "identifiers": ("id", "nss", "identifier"),
+        "identifier_types": ("idtype", "id_type", "nid", "identifier_type"),
+        "issue_number": ("number",),
+        "locations": ("location",),
+        "monochrome": gen_multipart_field_aliases("black_and_white"),
+        "name": ("title",),
+        "original_format": ("format",),
+        "page_count": ("pages",),
+        "reading_direction": ("direction",),
+        "path": (
+            "filename",
+            *gen_multipart_field_aliases("folders"),
+        ),
+        "series_groups": gen_multipart_field_aliases("series_groups"),
+        "scan_info": ("scan",),
+        "stories": ("story",),
+        "story_arcs": gen_multipart_field_aliases("story_arcs"),
+        "summary": ("comments", "description"),
+        "tags": ("tag",),
+        "teams": ("team",),
+        "updated_at": ("updated",),
+    }
+)
+
+
+ALIAS_FIELD_MAP = MappingProxyType(
+    {value: key for key, values in FIELDMAP.items() for value in values}
+)
+FIELD_TYPE_MAP = MappingProxyType(
+    {
+        **{field.name: field.__class__ for field in Comic._meta.get_fields()},
+        "role": ManyToManyField,
+        "issue": CharField,
+    }
+)

codex/views/browser/filters/search/field/__init__.py

@@ -0,0 +1 @@
+"""Column Lookup Queries from field tokens."""

codex/views/browser/filters/search/field/column.py

@@ -0,0 +1,53 @@
+"""Parse the field side of a field lookup."""
+
+from types import MappingProxyType
+
+from django.db.models import (
+    CharField,
+)
+from django.db.models.fields.related import ForeignKey, ManyToManyField
+
+from codex.logger.logging import get_logger
+from codex.views.browser.filters.search.aliases import FIELD_TYPE_MAP
+
+_FIELD_TO_REL_SPAN_MAP = MappingProxyType(
+    {
+        "role": "contributors__role__name",
+        "contributors": "contributors__person__name",
+        "identifier": "identifier__nss",
+        "identirier_type": "identifier__identifier_type__name",
+        "story_arcs": "story_arc_number__story_arc__name",
+    }
+)
+
+LOG = get_logger(__name__)
+
+
+def _parse_field_rel(field_name, rel_class):
+    """Set rel to comic attribute or relation span."""
+    rel = _FIELD_TO_REL_SPAN_MAP.get(field_name, "")
+    if not rel and rel_class in (ForeignKey, ManyToManyField):
+        # This must be after the special span maps
+        rel = f"{field_name}__name"
+
+    if rel.endswith("name"):
+        rel_class = CharField
+
+    if not rel:
+        # Comic attribute
+        rel = field_name
+
+    return rel_class, rel
+
+
+def parse_field(field_name: str):
+    """Parse the field size of the query in to database relations."""
+    rel_class = FIELD_TYPE_MAP.get(field_name)
+    if not rel_class:
+        reason = f"Unknown field specified in search query {field_name}"
+        raise ValueError(reason)
+
+    many_to_many = rel_class == ManyToManyField
+
+    rel_class, rel = _parse_field_rel(field_name, rel_class)
+    return rel_class, rel, many_to_many

codex/views/browser/filters/search/field/expression.py

@@ -0,0 +1,177 @@
+"""Parse field lookup right hand side expression."""
+
+import re
+from decimal import Decimal
+from types import MappingProxyType
+
+from comicbox.fields.fields import IssueField
+from dateparser import parse
+from django.db.models import (
+    BooleanField,
+    CharField,
+    DateField,
+    DateTimeField,
+    TextField,
+)
+from django.db.models.fields import DecimalField, PositiveSmallIntegerField
+from humanfriendly import parse_size
+
+from codex.logger.logging import get_logger
+from codex.settings.settings import FALSY
+
+_QUOTES_RE = re.compile(r"[\"']")
+_OP_MAP = MappingProxyType({">": "gt", ">=": "gte", "<": "lt", "<=": "lte"})
+_RANGE_RE = re.compile(r"\.{2,}")
+_PARSE_ISSUE_MATCHER = re.compile(r"(?P<issue_number>\d*\.?\d*)(?P<issue_suffix>.*)")
+_LIKE_RE = re.compile(f"([{re.escape('%_')}])")
+LOG = get_logger(__name__)
+
+
+def _parse_issue_value(value):
+    """Parse a compound issue value into number & suffix."""
+    value = IssueField.parse_issue(value)
+    if not value:
+        return None, None
+    matches = _PARSE_ISSUE_MATCHER.match(value)
+    if not matches:
+        return None, None
+    numeric_value = Decimal(matches.group("issue_number"))
+    suffix_value = matches.group("issue_suffix")
+    return numeric_value, suffix_value
+
+
+def _parse_issue_values(rel, value, to_value=None):
+    """Issue is not a column. Convert to issue_number and issue_suffix."""
+    numeric_value, suffix_value = _parse_issue_value(value)
+    if to_value is not None:
+        to_numeric_value, to_suffix_value = _parse_issue_value(to_value)
+    else:
+        to_numeric_value = to_suffix_value = None
+
+    q_dict = {}
+
+    if numeric_value is not None:
+        issue_number_field = rel.replace("issue", "issue_number")
+        if to_numeric_value is not None:
+            numeric_value = (numeric_value, to_numeric_value)
+        q_dict[issue_number_field] = numeric_value
+
+    if suffix_value is not None:
+        issue_suffix_field = rel.replace("issue", "issue_suffix")
+        if to_suffix_value is not None:
+            suffix_value = (suffix_value, to_suffix_value)
+        q_dict[issue_suffix_field] = suffix_value
+
+    return q_dict
+
+
+def _cast_value(rel, rel_class, value):
+    """Cast values by relation class."""
+    if rel.startswith("size"):
+        value = parse_size(value)
+    elif rel_class == PositiveSmallIntegerField:
+        value = int(value)
+    elif rel_class == DecimalField:
+        value = Decimal(value)
+    elif rel_class == BooleanField:
+        value = value not in FALSY
+    elif rel_class in (DateField, DateTimeField):
+        value = parse(value)
+        if rel_class == DateField and value:
+            value = value.date()
+    # elif rel_class in (CharField, TextField):
+    #    value = value
+    return value
+
+
+def _parse_operator_numeric(rel, rel_class, value):
+    value = _cast_value(rel, rel_class, value)
+    if value is None:
+        return {}
+    return {rel: value}
+
+
+def _parse_operator_text(rel, exp):
+    """Parse text value operators."""
+    if rel == "issue":
+        return _parse_issue_values(rel, exp)
+
+    value = _glob_to_like(exp)
+    rel += "__like"
+    return {rel: value}
+
+
+def _parse_operator(operator, rel, rel_class, exp):
+    """Move value operator out of value into relation operator."""
+    lookup = _OP_MAP[operator]
+    span_rel = f"{rel}__{lookup}" if operator else rel
+    value = exp[len(operator) :]
+    if rel == "issue":
+        return _parse_issue_values(span_rel, value)
+    return _parse_operator_numeric(span_rel, rel_class, value)
+
+
+def _parse_operator_range(rel, rel_class, value) -> dict:
+    """Parse range operator."""
+    range_from_value, range_to_value = _RANGE_RE.split(value, 1)
+    rel = f"{rel}__range"
+    if rel == ("issue__range"):
+        return _parse_issue_values(
+            rel,
+            range_from_value,
+            range_to_value,
+        )
+    range_value = (
+        _cast_value(rel, rel_class, range_from_value),
+        _cast_value(rel, rel_class, range_to_value),
+    )
+    return {rel: range_value}
+
+
+def _glob_to_like(value) -> str:
+    """Transform globs into like lookups."""
+    # Escape like tokens
+    value = _LIKE_RE.sub(r"\\\g<1>", value)
+
+    # Remove double stars
+    while True:
+        new_value = value.replace("**", "*")
+        if new_value == value:
+            value = new_value
+            break
+        value = new_value
+
+    if not value:
+        return "%"
+
+    # Replace starts with like operands
+    value = value.replace("*", "%")
+
+    # If not a prefix or suffix search, make a like term search
+    if not value.startswith("%") and not value.endswith("%"):
+        value = f"%{value}%"
+
+    return value
+
+
+def parse_expression(rel, rel_class, exp) -> dict:
+    """Parse the operators of the value size of the field query."""
+    exp = _QUOTES_RE.sub("", exp)
+
+    for op in _OP_MAP:
+        if exp.startswith(op):
+            q_dict = _parse_operator(
+                op,
+                rel,
+                rel_class,
+                exp,
+            )
+            break
+    else:
+        if ".." in exp:
+            q_dict = _parse_operator_range(rel, rel_class, exp)
+        elif rel_class in (CharField, TextField) and not rel.startswith("volume"):
+            q_dict = _parse_operator_text(rel, exp)
+        else:
+            q_dict = _parse_operator_numeric(rel, rel_class, exp)
+    return q_dict

codex/views/browser/filters/search/field/filter.py

@@ -0,0 +1,70 @@
+"""Parse the browser query by removing field queries and doing them with the ORM."""
+
+from django.db.models import Q
+
+from codex.logger.logging import get_logger
+from codex.views.browser.filters.field import ComicFieldFilterView
+from codex.views.browser.filters.search.field.column import parse_field
+from codex.views.browser.filters.search.field.parse import get_field_query
+
+LOG = get_logger(__name__)
+
+
+class BrowserFieldQueryFilter(ComicFieldFilterView):
+    """Parse the browser query by removing field queries and doing them with the ORM."""
+
+    @staticmethod
+    def _combine_q(q, other_q, op):
+        if isinstance(other_q, tuple):
+            rel, val = other_q
+            if rel.endswith("__like") and val == "%":
+                # Remove likes that would match everything
+                return q
+            other_q = Q(**{rel: val})
+        if op == Q.AND:
+            q &= other_q
+        else:
+            q |= other_q
+        return q
+
+    @classmethod
+    def _hoist_filters(cls, filter_q_list, exclude_q_list, new_q):
+        """Peel top layer of queries into multiple filter and exclude clauses."""
+        # This makes m2m queries behave more as expected and may optimize fk queries.
+        if new_q.connector == Q.AND:
+            for child in new_q.children:
+                if isinstance(child, Q) and child.negated:
+                    child.negated = False
+                    q = cls._combine_q(Q(), child, new_q.connector)
+                    exclude_q_list.append(q)
+                else:
+                    q = cls._combine_q(Q(), child, new_q.connector)
+                    filter_q_list.append(q)
+        else:
+            filter_q_list[0] = cls._combine_q(filter_q_list[0], new_q, new_q.connector)
+
+    def _parse_field_query(self, col, exp, model, filter_q_list, exclude_q_list):
+        try:
+            rel_class, rel, many_to_many = parse_field(col)
+
+            if q := get_field_query(rel, rel_class, exp, model, many_to_many):
+                self._hoist_filters(filter_q_list, exclude_q_list, q)
+        except Exception as exc:
+            token = f"{col}:{exp}"
+            msg = f"Parsing field query {token} - {exc}"
+            LOG.warning(msg)
+            self.search_error = msg
+
+    def get_search_field_filters(self, model, field_token_pairs):
+        """Parse and apply field query filters."""
+        filter_q_list = []
+        exclude_q_list = []
+        if not field_token_pairs:
+            return filter_q_list, exclude_q_list
+
+        filter_q_list.append(Q())
+        exclude_q_list.append(Q())
+        for col, exp in field_token_pairs:
+            self._parse_field_query(col, exp, model, filter_q_list, exclude_q_list)
+
+        return filter_q_list, exclude_q_list

codex/views/browser/filters/search/field/optimize.py

@@ -0,0 +1,71 @@
+"""Optimize Text Lookups."""
+# XXX Unused XXX multiple like queries test better
+
+import re
+
+from django.db.models import Q
+
+# Just an unlikely character
+_PERCENT_PLACEHOLDER = "Ɍ"
+
+
+def _like_to_regex(like):
+    """Transform a glob into a safe regex for sqlite3."""
+    # unescape like
+    like = like.replace(r"\%", _PERCENT_PLACEHOLDER)
+    like = like.replace(r"\_", "_")
+
+    while like[0] == "%" and like[-1] == "%":
+        like = like[1:-1]
+
+    parts = like
+
+    prefix = suffix = ""
+
+    if parts[0] == "%":
+        parts = parts[1:]
+        suffix = "$"
+    elif parts[-1] == "%":
+        parts = parts[:-1]
+        prefix = "^"
+
+    if not parts:
+        return parts
+
+    regex = parts.replace(_PERCENT_PLACEHOLDER, "%")
+    regex = re.escape(regex)
+    regex = regex.replace("%", ".*")
+
+    return prefix + regex + suffix
+
+
+def _regex_like(regex, lookahead):
+    """Bracket a regex with any characters if positive lookahead."""
+    if lookahead == "=":
+        regex = ".*" + regex + ".*"
+    return regex
+
+
+def like_qs_to_regex_q(q: Q, regex_op: str, many_to_many: bool):
+    """Optimize a tree of like lookup qs to one regex q."""
+    regexes = []
+    # Optimize like equations into regexes
+    rel = ""
+
+    if many_to_many:
+        regex_op = "|"
+
+    lookahead = "!" if q.negated else ":" if regex_op == "|" else "="
+    for child_q in q.children:
+        rel, like = child_q
+        value = _like_to_regex(like)
+        value = _regex_like(value, lookahead)
+        regex = rf"(?{lookahead}{value})"
+        regexes.append(regex)
+
+    regex_value = regex_op.join(regexes)
+    regex_value = _regex_like(regex_value, lookahead)
+
+    rel = rel.replace("like", "iregex")
+
+    return Q(**{rel: regex_value})