codex 1.6.19__py3-none-any.whl → 1.7.0__py3-none-any.whl

codex/views/browser/filters/search/field/filter.py

@@ -0,0 +1,70 @@
+"""Parse the browser query by removing field queries and doing them with the ORM."""
+
+from django.db.models import Q
+
+from codex.logger.logging import get_logger
+from codex.views.browser.filters.field import ComicFieldFilterView
+from codex.views.browser.filters.search.field.column import parse_field
+from codex.views.browser.filters.search.field.parse import get_field_query
+
+LOG = get_logger(__name__)
+
+
+class BrowserFieldQueryFilter(ComicFieldFilterView):
+    """Parse the browser query by removing field queries and doing them with the ORM."""
+
+    @staticmethod
+    def _combine_q(q, other_q, op):
+        if isinstance(other_q, tuple):
+            rel, val = other_q
+            if rel.endswith("__like") and val == "%":
+                # Remove likes that would match everything
+                return q
+            other_q = Q(**{rel: val})
+        if op == Q.AND:
+            q &= other_q
+        else:
+            q |= other_q
+        return q
+
+    @classmethod
+    def _hoist_filters(cls, filter_q_list, exclude_q_list, new_q):
+        """Peel top layer of queries into multiple filter and exclude clauses."""
+        # This makes m2m queries behave more as expected and may optimize fk queries.
+        if new_q.connector == Q.AND:
+            for child in new_q.children:
+                if isinstance(child, Q) and child.negated:
+                    child.negated = False
+                    q = cls._combine_q(Q(), child, new_q.connector)
+                    exclude_q_list.append(q)
+                else:
+                    q = cls._combine_q(Q(), child, new_q.connector)
+                    filter_q_list.append(q)
+        else:
+            filter_q_list[0] = cls._combine_q(filter_q_list[0], new_q, new_q.connector)
+
+    def _parse_field_query(self, col, exp, model, filter_q_list, exclude_q_list):
+        try:
+            rel_class, rel, many_to_many = parse_field(col)
+
+            if q := get_field_query(rel, rel_class, exp, model, many_to_many):
+                self._hoist_filters(filter_q_list, exclude_q_list, q)
+        except Exception as exc:
+            token = f"{col}:{exp}"
+            msg = f"Parsing field query {token} - {exc}"
+            LOG.warning(msg)
+            self.search_error = msg
+
+    def get_search_field_filters(self, model, field_token_pairs):
+        """Parse and apply field query filters."""
+        filter_q_list = []
+        exclude_q_list = []
+        if not field_token_pairs:
+            return filter_q_list, exclude_q_list
+
+        filter_q_list.append(Q())
+        exclude_q_list.append(Q())
+        for col, exp in field_token_pairs:
+            self._parse_field_query(col, exp, model, filter_q_list, exclude_q_list)
+
+        return filter_q_list, exclude_q_list

codex/views/browser/filters/search/field/optimize.py

@@ -0,0 +1,71 @@
+"""Optimize Text Lookups."""
+# XXX Unused XXX multiple like queries test better
+
+import re
+
+from django.db.models import Q
+
+# Just an unlikely character
+_PERCENT_PLACEHOLDER = "Ɍ"
+
+
+def _like_to_regex(like):
+    """Transform a glob into a safe regex for sqlite3."""
+    # unescape like
+    like = like.replace(r"\%", _PERCENT_PLACEHOLDER)
+    like = like.replace(r"\_", "_")
+
+    while like[0] == "%" and like[-1] == "%":
+        like = like[1:-1]
+
+    parts = like
+
+    prefix = suffix = ""
+
+    if parts[0] == "%":
+        parts = parts[1:]
+        suffix = "$"
+    elif parts[-1] == "%":
+        parts = parts[:-1]
+        prefix = "^"
+
+    if not parts:
+        return parts
+
+    regex = parts.replace(_PERCENT_PLACEHOLDER, "%")
+    regex = re.escape(regex)
+    regex = regex.replace("%", ".*")
+
+    return prefix + regex + suffix
+
+
+def _regex_like(regex, lookahead):
+    """Bracket a regex with any characters if positive lookahead."""
+    if lookahead == "=":
+        regex = ".*" + regex + ".*"
+    return regex
+
+
+def like_qs_to_regex_q(q: Q, regex_op: str, many_to_many: bool):
+    """Optimize a tree of like lookup qs to one regex q."""
+    regexes = []
+    # Optimize like equations into regexes
+    rel = ""
+
+    if many_to_many:
+        regex_op = "|"
+
+    lookahead = "!" if q.negated else ":" if regex_op == "|" else "="
+    for child_q in q.children:
+        rel, like = child_q
+        value = _like_to_regex(like)
+        value = _regex_like(value, lookahead)
+        regex = rf"(?{lookahead}{value})"
+        regexes.append(regex)
+
+    regex_value = regex_op.join(regexes)
+    regex_value = _regex_like(regex_value, lookahead)
+
+    rel = rel.replace("like", "iregex")
+
+    return Q(**{rel: regex_value})

codex/views/browser/filters/search/field/parse.py

@@ -0,0 +1,187 @@
+"""Parse field boolean expressions into Django ORM Queries."""
+
+import re
+from typing import TYPE_CHECKING
+
+from django.db.models import Q
+from pyparsing import (
+    CaselessLiteral,
+    OpAssoc,
+    ParserElement,
+    QuotedString,
+    Word,
+    infix_notation,
+    printables,
+)
+
+from codex.models.base import MAX_NAME_LEN
+from codex.models.comic import Comic
+from codex.views.browser.filters.search.field.expression import parse_expression
+
+if TYPE_CHECKING:
+    from pyparsing.helpers import InfixNotationOperatorSpec
+
+_QUOTES_REXP = r"(?:\".*?\")"
+_OPERATORS_REXP = "|".join(("and not", "or not", "and", "or"))
+_BEGIN_NOT_REXP = r"^\s*\(?\s*(?P<not>not)"
+_IMPLICIT_AND_REXP = (
+    rf"{_QUOTES_REXP}|\ (?P<ok>{_OPERATORS_REXP})\ |(?P<bare>(?:\ not)?\ )\S"
+)
+_BEGIN_NOT_RE = re.compile(_BEGIN_NOT_REXP, flags=re.IGNORECASE)
+_IMPLICIT_AND_RE = re.compile(_IMPLICIT_AND_REXP, flags=re.IGNORECASE)
+ParserElement.enablePackrat()
+
+
+class BoolOperand:
+    """Hacky Base for injecting rel."""
+
+    def __init__(self, tokens, context):
+        """Initialize value from first token."""
+        self.value = tokens[0]
+        self.context = context
+
+    def __repr__(self) -> str:
+        """Represent as string."""
+        return str(self.value)
+
+    def _prefix_q_dict(self, q_dict):
+        """Add (or subtract!) relation prefixes to q_dict for the model."""
+        model = self.context[2]
+        prefix = "" if model == Comic else "comic__"
+        model_span = model.__name__.lower() + "__"
+        prefixed_q_dict = {}
+        for parsed_rel, value in q_dict.items():
+            prefixed_rel = (
+                parsed_rel.removeprefix(model_span)
+                if parsed_rel.startswith(model_span)
+                else prefix + parsed_rel
+            )
+            prefixed_q_dict[prefixed_rel] = value
+        return prefixed_q_dict
+
+    def to_query(self) -> Q:
+        """Construct Django ORM Query from rel & value."""
+        rel = self.context[0]
+        rel_class = self.context[1]
+        q_dict = parse_expression(rel, rel_class, self.value)
+        if not q_dict:
+            return Q()
+
+        prefixed_q_dict = self._prefix_q_dict(q_dict)
+
+        return Q(**prefixed_q_dict)
+
+
+class BoolNot(BoolOperand):
+    """NOT Operand."""
+
+    def __init__(self, tokens, context):
+        """Initialize args from first token."""
+        self.arg = tokens[0][1]
+        self.context = context
+
+    def __repr__(self) -> str:
+        """Represent as string."""
+        return f"NOT {self.arg}"
+
+    def to_query(self) -> Q:
+        """Negate argument query."""
+        q = self.arg.to_query()
+        return ~q
+
+
+class BoolBinaryOperand:
+    """Boolean Binary Operand."""
+
+    OP: str = ""
+
+    def __init__(self, tokens, context):
+        """Initialize args from first two tokens."""
+        self.args = tokens[0][0::2]
+        self.context = context
+
+    def __repr__(self) -> str:
+        """Represent as string."""
+        sep = f" {self.OP} "
+        return f"({sep.join(map(str, self.args))})"
+
+    def to_query(self) -> Q:
+        """Construct Django ORM Query from args."""
+        q = Q()
+
+        for arg in self.args:
+            arg_q = arg.to_query()
+            if self.OP == Q.AND:
+                q &= arg_q
+            else:
+                q |= arg_q
+
+        return q
+
+
+class BoolAnd(BoolBinaryOperand):
+    """AND Operand."""
+
+    OP = Q.AND
+
+
+class BoolOr(BoolBinaryOperand):
+    """OR Operand."""
+
+    OP = Q.OR
+
+
+def _get_context_operand(op_class, context):
+    """Hack context into operands."""
+
+    def parse_action(_s, _loc, toks):
+        """Inject context into operand classes."""
+        return op_class(toks, context)
+
+    return parse_action
+
+
+def _create_context_expression(context):
+    # I can't find a way for pyparsing to inject context after the grammar is defined.
+    bool_operand_fn = _get_context_operand(BoolOperand, context)
+    bool_not_operand_fn = _get_context_operand(BoolNot, context)
+    bool_and_operand_fn = _get_context_operand(BoolAnd, context)
+    bool_or_operand_fn = _get_context_operand(BoolOr, context)
+
+    # Order is important for operands so quoted strings get parsed first
+    bool_operand = QuotedString('"') | Word(
+        printables, exclude_chars="(),", max=MAX_NAME_LEN
+    )
+    bool_operand.set_parse_action(bool_operand_fn)
+    bool_operand.set_name("boolean_operand")
+    op_list: list[InfixNotationOperatorSpec] = [
+        # In order of precedence
+        (CaselessLiteral("not"), 1, OpAssoc.RIGHT, bool_not_operand_fn),
+        (CaselessLiteral("and"), 2, OpAssoc.LEFT, bool_and_operand_fn),
+        (CaselessLiteral("or"), 2, OpAssoc.LEFT, bool_or_operand_fn),
+    ]
+
+    bool_expr = infix_notation(bool_operand, op_list)
+    bool_expr.set_name("boolean_expression")
+    return bool_expr
+
+
+def get_field_query(rel, rel_class, exp, model, many_to_many):
+    """Convert rel and text expression into queries."""
+    # Allow negative column search
+    begin_not_match = _BEGIN_NOT_RE.search(exp)
+    if begin_not_match:
+        start = begin_not_match.start("not")
+        exp = exp[:start] + '"" and ' + exp[start:]
+
+    # Add implicit and for the parser
+    exp = _IMPLICIT_AND_RE.sub(
+        lambda m: f" and{m.group(0)}" if m.group("bare") else m.group(0), exp
+    )
+
+    context = rel, rel_class, model, many_to_many
+    bool_expr = _create_context_expression(context)
+
+    parsed_result = bool_expr.parse_string(exp)
+    root_bool_operand: BoolOperand = parsed_result[0]  # type:ignore
+    return root_bool_operand.to_query()

codex/views/browser/filters/search/fts.py

@@ -0,0 +1,24 @@
+"""Search Filters Methods."""
+
+from codex.logger.logging import get_logger
+from codex.views.browser.filters.search.field.filter import BrowserFieldQueryFilter
+
+LOG = get_logger(__name__)
+
+
+class BrowserFTSFilter(BrowserFieldQueryFilter):
+    """Search Filters Methods."""
+
+    def get_fts_filter(self, model, text):
+        """Perform the search and return the scores as a dict."""
+        fts_filter = {}
+        try:
+            if text:
+                rel = self.get_rel_prefix(model)
+                # Custom lookup defined in codex.models
+                rel += "comicfts__match"
+                fts_filter[rel] = text
+        except Exception:
+            LOG.exception("Getting Full Text Search Filter.")
+            self.search_error = "Error creating full text search filter"
+        return fts_filter

codex/views/browser/filters/search/parse.py

@@ -0,0 +1,250 @@
+"""Search Filters Methods."""
+
+import re
+from types import MappingProxyType
+
+from django.db.models.query import Q
+
+from codex.logger.logging import get_logger
+from codex.models import AdminFlag
+from codex.models.comic import ComicFTS
+from codex.views.browser.filters.search.aliases import ALIAS_FIELD_MAP
+from codex.views.browser.filters.search.fts import BrowserFTSFilter
+from codex.views.const import MAX_OBJ_PER_PAGE
+
+LOG = get_logger(__name__)
+_FTS_COLUMNS = frozenset(
+    {field.name for field in ComicFTS._meta.get_fields()}
+    - {"comic", "updated_at", "created_at"}
+)
+_NON_FTS_COLUMNS = frozenset(
+    {
+        "volume",
+        "created_at",
+        "updated_at",
+        "issue_number",
+        "issue_suffix",
+        "year",
+        "month",
+        "day",
+        "community_rating",
+        "criticial_rating",
+        "page_count",
+        "monochrome",
+        "date",
+        "decade",
+        "size",
+        "path",
+        "identifier",
+        "identifier_type",
+    }
+)
+_VALID_COLUMNS = frozenset(_FTS_COLUMNS | _NON_FTS_COLUMNS)
+_QUOTES_REXP = r"\".*?\""
+_COLUMN_EXPRESSION_OPERATORS_REXP = (
+    rf"(?:{_QUOTES_REXP})|(?P<star>\B[\*\<\>]\w|\.{2,}|\w\*\w)"
+)
+_COLUMN_EXPRESSION_OPERATORS_RE = re.compile(_COLUMN_EXPRESSION_OPERATORS_REXP)
+_FTS_OPERATORS = frozenset({"and", "not", "or", "near"})
+_FTS_OPERATOR_REXP = rf"(?P<operator>\b{'|'.join(_FTS_OPERATORS)}\b)"
+_FTS_OPERATOR_RE = re.compile(_FTS_OPERATOR_REXP, flags=re.IGNORECASE)
+_MULTI_COL_REXP = r"(?P<multi_col>\{.*?\})"
+_SINGLE_COL_REXP = r"(?P<col>[a-z_]+)"
+_EXP_REXP = rf"\s*(?P<exp>\(.*?\)|{_QUOTES_REXP}|\S+)"
+_COL_REXP = rf"({_MULTI_COL_REXP}|{_SINGLE_COL_REXP}):{_EXP_REXP}"
+_TOKEN_PRE_OP_REXP = r"(?:(?P<preop>and|or|not)\s+)?"  # noqa: S105
+_TOKEN_REXP = rf"(?P<token>{_TOKEN_PRE_OP_REXP}{_COL_REXP}|\S+)"
+_TOKEN_RE = re.compile(_TOKEN_REXP, flags=re.IGNORECASE)
+
+
+class SearchFilterView(BrowserFTSFilter):
+    """Search Query Parser."""
+
+    ADMIN_FLAG_VALUE_KEY_MAP = MappingProxyType(
+        {
+            AdminFlag.FlagChoices.FOLDER_VIEW.value: "folder_view",
+        }
+    )
+
+    def __init__(self, *args, **kwargs):
+        """Initialize search variables."""
+        super().__init__(*args, **kwargs)
+        self._admin_flags: MappingProxyType[str, bool] | None = None
+        self.fts_mode = False
+        self.search_mode = False
+        self.search_error = ""
+
+    @property
+    def admin_flags(self) -> MappingProxyType[str, bool]:
+        """Set browser relevant admin flags."""
+        if self._admin_flags is None:
+            if self.ADMIN_FLAG_VALUE_KEY_MAP:
+                admin_pairs = AdminFlag.objects.filter(
+                    key__in=self.ADMIN_FLAG_VALUE_KEY_MAP.keys()
+                ).values_list("key", "on")
+            else:
+                admin_pairs = ()
+            admin_flags = {}
+            for key, on in admin_pairs:
+                export_key = self.ADMIN_FLAG_VALUE_KEY_MAP[key]
+                admin_flags[export_key] = on
+            self._admin_flags = MappingProxyType(admin_flags)
+        return self._admin_flags
+
+    def _is_path_column_allowed(self):
+        """Is path column allowed."""
+        return self.is_admin or bool(self.admin_flags["folder_view"])
+
+    @staticmethod
+    def _is_column_operators_used(exp) -> bool:
+        """Detect column expression operators, but not inside quotes."""
+        for match in _COLUMN_EXPRESSION_OPERATORS_RE.finditer(exp):
+            if match.group("star"):
+                return True
+        return False
+
+    def _parse_column_match(self, preop, col, exp, field_tokens):  # , fts_tokens):
+        col = ALIAS_FIELD_MAP.get(col, col)
+        if col not in _VALID_COLUMNS:
+            return True
+        if col == "path" and not self._is_path_column_allowed():
+            return True
+        if col in _NON_FTS_COLUMNS or self._is_column_operators_used(exp):
+            if not preop:
+                preop = "and"
+            if preop not in field_tokens:
+                field_tokens[preop] = set()
+            field_tokens[preop].add((col, exp))
+            return True
+
+        return False
+
+    @staticmethod
+    def _add_fts_token(fts_tokens, token):
+        token = _FTS_OPERATOR_RE.sub(lambda op: op.group("operator").upper(), token)
+        if (
+            token.lower() not in _FTS_OPERATORS
+            and not (token.startswith('"') and token.endswith('"'))
+            and ":" not in token
+        ):
+            token = f'"{token}"'
+        fts_tokens.append(token)
+
+    def _preparse_search_query_token(self, match, field_tokens, fts_tokens):
+        token = match.group("token")
+        if not token:
+            return
+
+        multi_col = match.group("multi_col")
+        col = match.group("col")
+        exp = match.group("exp")
+        if multi_col or not col or not exp:
+            # I could add multi-col to field groups, but nobody will care.
+            self._add_fts_token(fts_tokens, token)
+            return
+
+        preop = match.group("preop")
+        if not self._parse_column_match(preop, col, exp, field_tokens):
+            self._add_fts_token(fts_tokens, token)
+
+    def _preparse_search_query(self):
+        """Preparse search fields out of query text."""
+        text = self.params.get("q")  # type: ignore
+        field_tokens = {}
+        if not text:
+            return field_tokens, text
+
+        fts_tokens = []
+        for match in _TOKEN_RE.finditer(text):
+            try:
+                self._preparse_search_query_token(match, field_tokens, fts_tokens)
+            except Exception as exc:
+                tok = match.group(0) if match else "<unmatched>"
+                LOG.debug(f"Error preparsing search query token {tok}: {exc}")
+                self.search_error = "Syntax error"
+        text = " ".join(fts_tokens)
+
+        return field_tokens, text
+
+    def _create_search_filters(self, model):
+        field_tokens_dict, fts_text = self._preparse_search_query()
+        field_filter_q_list = []
+        field_exclude_q_list = []
+        for preop, field_token_pairs in field_tokens_dict.items():
+            if preop == "not":
+                exclude_q_list, include_q_list = self.get_search_field_filters(
+                    model, field_token_pairs
+                )
+            else:
+                # AND and OR
+                # XXX cannot do OR queries with MATCH, it decontextualizes MATCH somehow.
+                if preop == "or":
+                    self.search_error = "OR preceding column tokens with operator expressions will act as AND"
+                include_q_list, exclude_q_list = self.get_search_field_filters(
+                    model, field_token_pairs
+                )
+            field_filter_q_list += include_q_list
+            field_exclude_q_list += exclude_q_list
+
+        fts_filter_dict = self.get_fts_filter(model, fts_text)
+        if fts_filter_dict:
+            self.fts_mode = True
+            fts_q = Q(**fts_filter_dict)
+        else:
+            fts_q = Q()
+
+        return field_filter_q_list, field_exclude_q_list, fts_q
+
+    def _create_search_filter(self, filter_list):
+        """Apply search filter lists. Separate filter clauses are employed for m2m searches."""
+        combined_q = Q()
+        for q in filter_list:
+            if not q:
+                continue
+            combined_q &= q
+            self.search_mode = True
+
+        return combined_q
+
+    def get_search_filters(self, model):
+        """Preparse search, search and return the filter and scores."""
+        include_q = Q()
+        exclude_q = Q()
+        fts_q = Q()
+        try:
+            field_filter_q_list, field_exclude_q_list, fts_q = (
+                self._create_search_filters(model)
+            )
+            # Apply filters
+            include_q = self._create_search_filter(field_filter_q_list)
+            exclude_q = self._create_search_filter(field_exclude_q_list)
+
+        except Exception as exc:
+            msg = "Creating search filters"
+            LOG.exception(msg)
+            msg = f"{msg} - {exc}"
+            self.search_error = msg
+
+        return include_q, exclude_q, fts_q
+
+    def _is_search_results_limited(self) -> bool:
+        """Get search result limit from params."""
+        # user = self.request.user  # type: ignore
+        # if user and user.is_authenticated:
+        #    limited = bool(
+        #        self.params.get(  # type: ignore
+        #            "search_results_limit",
+        #            MAX_OBJ_PER_PAGE,
+        #        )
+        #    )
+        # else:
+        #    limited = True
+        # return limited
+        return True
+
+    def get_search_limit(self):
+        """Get search scores for choices and metadata."""
+        if not self.search_mode or not self._is_search_results_limited():
+            return 0
+        page = self.kwargs.get("page", 1)  # type: ignore
+        return page * MAX_OBJ_PER_PAGE + 1

codex/views/browser/group_mtime.py

@@ -0,0 +1,83 @@
+"""Group Mtime Function."""
+
+from logging import DEBUG, WARNING
+
+from django.db.models.aggregates import Aggregate, Max
+from django.db.models.functions import Greatest
+from django.db.utils import OperationalError
+
+from codex.logger.logging import get_logger
+from codex.models.functions import JsonGroupArray
+from codex.views.browser.filters.filter import BrowserFilterView
+from codex.views.const import EPOCH_START, EPOCH_START_DATETIMEFIELD, NONE_DATETIMEFIELD
+
+_FTS5_PREFIX = "fts5: "
+LOG = get_logger(__name__)
+
+
+class BrowserGroupMtimeView(BrowserFilterView):
+    """Annotations that also filter."""
+
+    def __init__(self, *args, **kwargs):
+        """Initialize memoized values."""
+        super().__init__(*args, **kwargs)
+        self._is_bookmark_filtered: bool | None = None
+
+    @property
+    def is_bookmark_filtered(self):
+        """Is bookmark filter in effect."""
+        if self._is_bookmark_filtered is None:
+            self._is_bookmark_filtered = bool(
+                self.params.get("filters", {}).get("bookmark")
+            )
+        return self._is_bookmark_filtered
+
+    def _handle_operational_error(self, err):
+        msg = err.args[0] if err.args else ""
+        if msg.startswith(_FTS5_PREFIX):
+            level = DEBUG
+            self.search_error = msg.removeprefix(_FTS5_PREFIX)
+        else:
+            level = WARNING
+            msg = str(err)
+        LOG.log(level, f"Query Error: {msg}")
+
+    def get_max_bookmark_updated_at_aggregate(
+        self, model, agg_func: type[Aggregate] = Max, default=NONE_DATETIMEFIELD
+    ):
+        """Get filtered maximum bookmark updated_at relation."""
+        bm_rel = self._get_bm_rel(model)
+        bm_filter = self._get_my_bookmark_filter(bm_rel)
+        bmua_rel = f"{bm_rel}__updated_at"
+        args = {"default": default, "filter": bm_filter}
+        if agg_func is JsonGroupArray:
+            args["distinct"] = True
+        return agg_func(bmua_rel, **args)
+
+    def get_group_mtime(self, model, group=None, pks=None, page_mtime=False):
+        """Get a filtered mtime for browser pages and mtime checker."""
+        qs = self.get_filtered_queryset(
+            model,
+            group=group,
+            pks=pks,
+            page_mtime=page_mtime,
+            bookmark_filter=self.is_bookmark_filtered,
+        )
+        mua = Max("updated_at", default=EPOCH_START_DATETIMEFIELD)
+        mbua = self.get_max_bookmark_updated_at_aggregate(
+            model, default=EPOCH_START_DATETIMEFIELD
+        )
+
+        try:
+            qs = qs.annotate(max=Greatest(mua, mbua))
+            # Forcing inner joins makes search work
+            # Can't run demote_joins on aggregate.
+            qs = self.force_inner_joins(qs)
+            first = qs.first()
+            mtime = first.max if first else EPOCH_START
+        except OperationalError as exc:
+            self._handle_operational_error(exc)
+            mtime = None
+        if mtime == NotImplemented:
+            mtime = None
+        return mtime

codex/views/browser/metadata/__init__.py

@@ -0,0 +1 @@
+"""Metadata View."""