codex 1.6.19__py3-none-any.whl → 1.7.0a0__py3-none-any.whl

codex/views/browser/filters/search/field/parse.py

@@ -0,0 +1,187 @@
+"""Parse field boolean expressions into Django ORM Queries."""
+
+import re
+from typing import TYPE_CHECKING
+
+from django.db.models import Q
+from pyparsing import (
+    CaselessLiteral,
+    OpAssoc,
+    ParserElement,
+    QuotedString,
+    Word,
+    infix_notation,
+    printables,
+)
+
+from codex.models.base import MAX_NAME_LEN
+from codex.models.comic import Comic
+from codex.views.browser.filters.search.field.expression import parse_expression
+
+if TYPE_CHECKING:
+    from pyparsing.helpers import InfixNotationOperatorSpec
+
+_QUOTES_REXP = r"(?:\".*?\")"
+_OPERATORS_REXP = "|".join(("and not", "or not", "and", "or"))
+_BEGIN_NOT_REXP = r"^\s*\(?\s*(?P<not>not)"
+_IMPLICIT_AND_REXP = (
+    rf"{_QUOTES_REXP}|\ (?P<ok>{_OPERATORS_REXP})\ |(?P<bare>(?:\ not)?\ )\S"
+)
+_BEGIN_NOT_RE = re.compile(_BEGIN_NOT_REXP, flags=re.IGNORECASE)
+_IMPLICIT_AND_RE = re.compile(_IMPLICIT_AND_REXP, flags=re.IGNORECASE)
+ParserElement.enablePackrat()
+
+
+class BoolOperand:
+    """Hacky Base for injecting rel."""
+
+    def __init__(self, tokens, context):
+        """Initialize value from first token."""
+        self.value = tokens[0]
+        self.context = context
+
+    def __repr__(self) -> str:
+        """Represent as string."""
+        return str(self.value)
+
+    def _prefix_q_dict(self, q_dict):
+        """Add (or subtract!) relation prefixes to q_dict for the model."""
+        model = self.context[2]
+        prefix = "" if model == Comic else "comic__"
+        model_span = model.__name__.lower() + "__"
+        prefixed_q_dict = {}
+        for parsed_rel, value in q_dict.items():
+            prefixed_rel = (
+                parsed_rel.removeprefix(model_span)
+                if parsed_rel.startswith(model_span)
+                else prefix + parsed_rel
+            )
+            prefixed_q_dict[prefixed_rel] = value
+        return prefixed_q_dict
+
+    def to_query(self) -> Q:
+        """Construct Django ORM Query from rel & value."""
+        rel = self.context[0]
+        rel_class = self.context[1]
+        q_dict = parse_expression(rel, rel_class, self.value)
+        if not q_dict:
+            return Q()
+
+        prefixed_q_dict = self._prefix_q_dict(q_dict)
+
+        return Q(**prefixed_q_dict)
+
+
+class BoolNot(BoolOperand):
+    """NOT Operand."""
+
+    def __init__(self, tokens, context):
+        """Initialize args from first token."""
+        self.arg = tokens[0][1]
+        self.context = context
+
+    def __repr__(self) -> str:
+        """Represent as string."""
+        return f"NOT {self.arg}"
+
+    def to_query(self) -> Q:
+        """Negate argument query."""
+        q = self.arg.to_query()
+        return ~q
+
+
+class BoolBinaryOperand:
+    """Boolean Binary Operand."""
+
+    OP: str = ""
+
+    def __init__(self, tokens, context):
+        """Initialize args from first two tokens."""
+        self.args = tokens[0][0::2]
+        self.context = context
+
+    def __repr__(self) -> str:
+        """Represent as string."""
+        sep = f" {self.OP} "
+        return f"({sep.join(map(str, self.args))})"
+
+    def to_query(self) -> Q:
+        """Construct Django ORM Query from args."""
+        q = Q()
+
+        for arg in self.args:
+            arg_q = arg.to_query()
+            if self.OP == Q.AND:
+                q &= arg_q
+            else:
+                q |= arg_q
+
+        return q
+
+
+class BoolAnd(BoolBinaryOperand):
+    """AND Operand."""
+
+    OP = Q.AND
+
+
+class BoolOr(BoolBinaryOperand):
+    """OR Operand."""
+
+    OP = Q.OR
+
+
+def _get_context_operand(op_class, context):
+    """Hack context into operands."""
+
+    def parse_action(_s, _loc, toks):
+        """Inject context into operand classes."""
+        return op_class(toks, context)
+
+    return parse_action
+
+
+def _create_context_expression(context):
+    # I can't find a way for pyparsing to inject context after the grammar is defined.
+    bool_operand_fn = _get_context_operand(BoolOperand, context)
+    bool_not_operand_fn = _get_context_operand(BoolNot, context)
+    bool_and_operand_fn = _get_context_operand(BoolAnd, context)
+    bool_or_operand_fn = _get_context_operand(BoolOr, context)
+
+    # Order is important for operands so quoted strings get parsed first
+    bool_operand = QuotedString('"') | Word(
+        printables, exclude_chars="(),", max=MAX_NAME_LEN
+    )
+    bool_operand.set_parse_action(bool_operand_fn)
+    bool_operand.set_name("boolean_operand")
+    op_list: list[InfixNotationOperatorSpec] = [
+        # In order of precedence
+        (CaselessLiteral("not"), 1, OpAssoc.RIGHT, bool_not_operand_fn),
+        (CaselessLiteral("and"), 2, OpAssoc.LEFT, bool_and_operand_fn),
+        (CaselessLiteral("or"), 2, OpAssoc.LEFT, bool_or_operand_fn),
+    ]
+
+    bool_expr = infix_notation(bool_operand, op_list)
+    bool_expr.set_name("boolean_expression")
+    return bool_expr
+
+
+def get_field_query(rel, rel_class, exp, model, many_to_many):
+    """Convert rel and text expression into queries."""
+    # Allow negative column search
+    begin_not_match = _BEGIN_NOT_RE.search(exp)
+    if begin_not_match:
+        start = begin_not_match.start("not")
+        exp = exp[:start] + '"" and ' + exp[start:]
+
+    # Add implicit and for the parser
+    exp = _IMPLICIT_AND_RE.sub(
+        lambda m: f" and{m.group(0)}" if m.group("bare") else m.group(0), exp
+    )
+
+    context = rel, rel_class, model, many_to_many
+    bool_expr = _create_context_expression(context)
+
+    parsed_result = bool_expr.parse_string(exp)
+    root_bool_operand: BoolOperand = parsed_result[0]  # type:ignore
+    return root_bool_operand.to_query()

codex/views/browser/filters/search/fts.py

@@ -0,0 +1,38 @@
+"""Search Filters Methods."""
+
+from django.db.models.query import Q
+
+from codex.logger.logging import get_logger
+from codex.models import Comic, StoryArc
+from codex.views.browser.filters.search.field.filter import BrowserFieldQueryFilter
+
+LOG = get_logger(__name__)
+
+
+class BrowserFTSFilter(BrowserFieldQueryFilter):
+    """Search Filters Methods."""
+
+    @staticmethod
+    def _get_fts_filter(model, text):
+        """Get the filter dict."""
+        prefix = (
+            ""
+            if model == Comic
+            else "storyarcnumber__comic__"
+            if model == StoryArc
+            else "comic__"
+        )
+        # Custom Lookup defined in codex.models
+        rel = prefix + "comicfts__match"
+        return {rel: text}
+
+    def get_fts_filter(self, model, text):
+        """Perform the search and return the scores as a dict."""
+        fts_filter = {}
+        try:
+            if text:
+                fts_filter = self._get_fts_filter(model, text)
+        except Exception:
+            LOG.exception("Getting Full Text Search Filter.")
+            self.search_error = "Error creating full text search filter"
+        return Q(**fts_filter)

codex/views/browser/filters/search/parse.py

@@ -0,0 +1,220 @@
+"""Search Filters Methods."""
+
+import re
+
+from codex.logger.logging import get_logger
+from codex.models.comic import ComicFTS
+from codex.views.browser.filters.search.aliases import ALIAS_FIELD_MAP
+from codex.views.browser.filters.search.fts import BrowserFTSFilter
+from codex.views.const import MAX_OBJ_PER_PAGE
+
+LOG = get_logger(__name__)
+_FTS_COLUMNS = frozenset(
+    {field.name for field in ComicFTS._meta.get_fields()}
+    - {"comic", "updated_at", "created_at"}
+)
+_NON_FTS_COLUMNS = frozenset(
+    {
+        "volume",
+        "created_at",
+        "updated_at",
+        "issue_number",
+        "issue_suffix",
+        "year",
+        "month",
+        "day",
+        "community_rating",
+        "criticial_rating",
+        "page_count",
+        "monochrome",
+        "date",
+        "decade",
+        "size",
+        "path",
+        "identifier",
+        "identifier_type",
+    }
+)
+_VALID_COLUMNS = frozenset(_FTS_COLUMNS | _NON_FTS_COLUMNS)
+_QUOTES_REXP = r"\".*?\""
+_COLUMN_EXPRESSION_OPERATORS_REXP = (
+    rf"(?:{_QUOTES_REXP})|(?P<star>\B[\*\<\>]\w|\.{2,}|\w\*\w)"
+)
+_COLUMN_EXPRESSION_OPERATORS_RE = re.compile(_COLUMN_EXPRESSION_OPERATORS_REXP)
+_FTS_OPERATORS = frozenset({"and", "not", "or", "near"})
+_FTS_OPERATOR_REXP = rf"(?P<operator>\b{'|'.join(_FTS_OPERATORS)}\b)"
+_FTS_OPERATOR_RE = re.compile(_FTS_OPERATOR_REXP, flags=re.IGNORECASE)
+_MULTI_COL_REXP = r"(?P<multi_col>\{.*?\})"
+_SINGLE_COL_REXP = r"(?P<col>[a-z_]+)"
+_EXP_REXP = rf"\s*(?P<exp>\(.*?\)|{_QUOTES_REXP}|\S+)"
+_COL_REXP = rf"({_MULTI_COL_REXP}|{_SINGLE_COL_REXP}):{_EXP_REXP}"
+_TOKEN_PRE_OP_REXP = r"(?:(?P<preop>and|or|not)\s+)?"  # noqa: S105
+_TOKEN_REXP = rf"(?P<token>{_TOKEN_PRE_OP_REXP}{_COL_REXP}|\S+)"
+_TOKEN_RE = re.compile(_TOKEN_REXP, flags=re.IGNORECASE)
+
+
+class SearchFilterView(BrowserFTSFilter):
+    """Search Query Parser."""
+
+    def __init__(self, *args, **kwargs):
+        """Initialize search variables."""
+        super().__init__(*args, **kwargs)
+        self.fts_mode = False
+        self.search_mode = False
+        self.search_error = ""
+
+    def _is_path_column_allowed(self):
+        """Is path column allowed."""
+        if not self.is_admin():  # type: ignore
+            if not "folder_view" not in self.admin_flags:  # type: ignore
+                # Ensure admin flags for Cover View
+                self.set_admin_flags()  # type: ignore
+            return bool(self.admin_flags.get("folder_view"))  # type: ignore
+        return True
+
+    @staticmethod
+    def _is_column_operators_used(exp):
+        """Detect column expression operators, but not inside quotes."""
+        for match in _COLUMN_EXPRESSION_OPERATORS_RE.finditer(exp):
+            if match.group("star"):
+                return True
+        return False
+
+    def _parse_column_match(self, preop, col, exp, field_tokens):  # , fts_tokens):
+        col = ALIAS_FIELD_MAP.get(col, col)
+        if col not in _VALID_COLUMNS:
+            return True
+        if col == "path" and not self._is_path_column_allowed():
+            return True
+        if col in _NON_FTS_COLUMNS or self._is_column_operators_used(exp):
+            if not preop:
+                preop = "and"
+            if preop not in field_tokens:
+                field_tokens[preop] = set()
+            field_tokens[preop].add((col, exp))
+            return True
+
+        return False
+
+    @staticmethod
+    def _add_fts_token(fts_tokens, token):
+        token = _FTS_OPERATOR_RE.sub(lambda op: op.group("operator").upper(), token)
+        if (
+            token.lower() not in _FTS_OPERATORS
+            and not (token.startswith('"') and token.endswith('"'))
+            and ":" not in token
+        ):
+            token = f'"{token}"'
+        fts_tokens.append(token)
+
+    def _preparse_search_query_token(self, match, field_tokens, fts_tokens):
+        token = match.group("token")
+        if not token:
+            return
+
+        multi_col = match.group("multi_col")
+        col = match.group("col")
+        exp = match.group("exp")
+        if multi_col or not col or not exp:
+            # I could add multi-col to field groups, but nobody will care.
+            self._add_fts_token(fts_tokens, token)
+            return
+
+        preop = match.group("preop")
+        if not self._parse_column_match(preop, col, exp, field_tokens):
+            self._add_fts_token(fts_tokens, token)
+
+    def _preparse_search_query(self):
+        """Preparse search fields out of query text."""
+        text = self.params.get("q")  # type: ignore
+        field_tokens = {}
+        if not text:
+            return field_tokens, text
+
+        fts_tokens = []
+        for match in _TOKEN_RE.finditer(text):
+            try:
+                self._preparse_search_query_token(match, field_tokens, fts_tokens)
+            except Exception as exc:
+                tok = match.group(0) if match else "<unmatched>"
+                LOG.debug(f"Error preparsing search query token {tok}: {exc}")
+                self.search_error = "Syntax error"
+        text = " ".join(fts_tokens)
+
+        return field_tokens, text
+
+    def _create_search_filters(self, model):
+        field_tokens_dict, fts_text = self._preparse_search_query()
+        field_filter_q_list = []
+        field_exclude_q_list = []
+        for preop, field_token_pairs in field_tokens_dict.items():
+            if preop == "not":
+                exclude_q_list, filter_q_list = self.get_search_field_filters(
+                    model, field_token_pairs
+                )
+            else:
+                # AND and OR
+                # XXX cannot do OR queries with MATCH, it decontextualizes MATCH somehow.
+                if preop == "or":
+                    self.search_error = "OR preceding column tokens with operator expressions will act as AND"
+                filter_q_list, exclude_q_list = self.get_search_field_filters(
+                    model, field_token_pairs
+                )
+            field_filter_q_list += filter_q_list
+            field_exclude_q_list += exclude_q_list
+        fts_filter = self.get_fts_filter(model, fts_text)
+        return field_exclude_q_list, field_filter_q_list, fts_filter
+
+    def _apply_search_filter_list(self, qs, filter_list, exclude):
+        """Apply search filter lists. Separate filter clauses are employed for m2m searches."""
+        for q in filter_list:
+            if not q:
+                continue
+            qs = qs.exclude(q) if exclude else qs.filter(q)
+            self.search_mode = True
+
+        return qs
+
+    def apply_search_filter(self, qs, model):
+        """Preparse search, search and return the filter and scores."""
+        try:
+            field_exclude_q_list, field_filter_q_list, fts_filter = (
+                self._create_search_filters(model)
+            )
+
+            # Apply filters
+            qs = self._apply_search_filter_list(qs, field_exclude_q_list, True)
+            qs = self._apply_search_filter_list(qs, field_filter_q_list, False)
+            if fts_filter:
+                qs = qs.filter(fts_filter)
+                self.search_mode = self.fts_mode = True
+
+        except Exception as exc:
+            msg = "Creating search filters"
+            LOG.exception(msg)
+            msg = f"{msg} - {exc}"
+            self.search_error = msg
+
+        return qs
+
+    def _is_search_results_limited(self) -> bool:
+        """Get search result limit from params."""
+        # user = self.request.user  # type: ignore
+        # if user and user.is_authenticated:
+        #    limited = bool(
+        #        self.params.get(  # type: ignore
+        #            "search_results_limit",
+        #            MAX_OBJ_PER_PAGE,
+        #        )
+        #    )
+        # else:
+        #    limited = True
+        # return limited
+        return True
+
+    def get_search_limit(self):
+        """Get search scores for choices and metadata."""
+        if not self.search_mode or not self._is_search_results_limited():
+            return 0
+        page = self.kwargs.get("page", 1)  # type: ignore
+        return page * MAX_OBJ_PER_PAGE + 1

codex/views/browser/order_by.py

@@ -13,8 +13,14 @@ class BrowserOrderByView(BrowserAnnotationsFilterView):
 
     def set_order_key(self):
         """Get the default order key for the view."""
-        self.order_key: str = self.params["order_by"]
-        order_reverse = self.params.get("order_reverse")
+        order_key: str = self.params["order_by"]
+        if order_key == "search_score" and not self.fts_mode:
+            # if no search scores, use sort_name, asc
+            order_key = "sort_name"
+            order_reverse = False
+        else:
+            order_reverse = self.params.get("order_reverse")
+        self.order_key = order_key
         self.order_agg_func = Max if order_reverse else Min
 
     def _add_comic_order_by(self, order_key, comic_sort_names):

codex/views/browser/page_in_bounds.py

@@ -0,0 +1,88 @@
+"""Browser Page Bounds Checking."""
+
+from codex.logger.logging import get_logger
+from codex.views.browser.annotations import BrowserAnnotationsView
+from codex.views.util import Route
+
+LOG = get_logger(__name__)
+
+
+class BrowserPageInBoundsView(BrowserAnnotationsView):
+    """Browser Page Bounds Checking."""
+
+    def _get_back_one_page_route(self, num_pages):
+        """Get max page if oob or 1."""
+        LOG.debug("Redirect back one page.")
+        group = self.kwargs.get("group")
+        pks = self.kwargs.get("pks")
+        page = self.kwargs.get("page", 1)
+        new_page = num_pages if num_pages and page > num_pages else 1
+        pks = pks if pks else "0"
+        return {"group": group, "pks": pks, "page": new_page}
+
+    def _get_up_breadcrumbs(self):
+        """Walk up the breadcrumbs to get the next level up."""
+        breadcrumbs = self.params.get("breadcrumbs", [])
+        new_breadcrumbs = []
+        level = False
+        group = self.kwargs.get("group")
+        pks = self.kwargs.get("pks")
+        page = self.kwargs.get("page", 1)
+        current_route = Route(group=group, pks=pks, page=page)
+        for crumb in reversed(breadcrumbs):
+            if not level:
+                crumb_route = Route(**crumb)
+                level = current_route & crumb_route
+                continue
+            new_breadcrumbs = [crumb, *new_breadcrumbs]
+
+        if not new_breadcrumbs:
+            if group not in ("f", "a"):
+                group = "r"
+            top_route = {"group": group, "pks": (), "page": 1}
+            new_breadcrumbs = [top_route]
+        return new_breadcrumbs
+
+    def _get_up_page_redirect(self):
+        """Walk up the breadcrumbs."""
+        try:
+            up_breadcrumbs = self._get_up_breadcrumbs()
+            route_mask = up_breadcrumbs[-1]
+            settings_mask = {"breadcrumbs": up_breadcrumbs}
+            LOG.debug("Redirect up a level.")
+        except IndexError:
+            group = self.kwargs.get("group")
+            pks = "0"
+            new_page = 1
+            route_mask = {"group": group, "pks": pks, "page": new_page}
+            settings_mask = {"breadcrumbs": [route_mask]}
+            LOG.debug("Redirect to all at current group.")
+        return route_mask, settings_mask
+
+    def _handle_page_out_of_bounds(self, num_pages):
+        """Handle out of bounds redirect."""
+        # Try to find a logical page to run to.
+        group = self.kwargs.get("group")
+        page = self.kwargs.get("page", 1)
+        pks = self.kwargs.get("pks")
+        reason = f"{group=} {pks=} {page=} does not exist."
+
+        # Adjust route mask for redirect
+        if num_pages and page > 1:
+            route_mask = self._get_back_one_page_route(num_pages)
+            settings_mask = None
+        else:
+            # This now only occurs when page < 1
+            route_mask, settings_mask = self._get_up_page_redirect()
+
+        self.raise_redirect(reason, route_mask=route_mask, settings_mask=settings_mask)
+
+    def check_page_in_bounds(self, num_pages: int):
+        """Redirect page out of bounds."""
+        page = self.kwargs.get("page", 1)
+        if page == 1 or (page >= 1 and page <= num_pages):
+            # Don't redirect if on the root page for the group.
+            # Or page within valid range.
+            return
+
+        self._handle_page_out_of_bounds(num_pages)