ckanext-search-tweaks 0.4.12__tar.gz → 0.6.0__tar.gz

{ckanext-search-tweaks-0.4.12/ckanext_search_tweaks.egg-info → ckanext-search-tweaks-0.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ckanext-search-tweaks
-Version: 0.4.12
+Version: 0.6.0
 Home-page: https://github.com/DataShades/ckanext-search-tweaks
 Author: Sergey Motornyuk
 Author-email: sergey.motornyuk@linkdigital.com.au
@@ -8,11 +8,14 @@ License: AGPL
 Keywords: CKAN
 Classifier: Development Status :: 4 - Beta
 Classifier: License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)
-Classifier: Programming Language :: Python :: 2.7
-Requires-Python: >=3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
 Description-Content-Type: text/markdown
-Provides-Extra: advanced-search
 License-File: LICENSE
+Requires-Dist: freezegun
+Requires-Dist: typing_extensions>=4.0.0
+Provides-Extra: advanced-search
 
 [![Tests](https://github.com/DataShades/ckanext-search-tweaks/actions/workflows/test.yml/badge.svg?branch=master)](https://github.com/DataShades/ckanext-search-tweaks/actions)
 

{ckanext-search-tweaks-0.4.12 → ckanext-search-tweaks-0.6.0}/ckanext/__init__.py

@@ -1,4 +1,3 @@
-# encoding: utf-8
 
 # this is a namespace package
 try:

ckanext-search-tweaks-0.6.0/ckanext/search_tweaks/__init__.py

@@ -0,0 +1,3 @@
+from __future__ import annotations
+
+from .shared import feature_disabled

{ckanext-search-tweaks-0.4.12 → ckanext-search-tweaks-0.6.0}/ckanext/search_tweaks/advanced_search/plugin.py

@@ -1,6 +1,8 @@
 from __future__ import annotations
+
 import json
 from typing import Any
+
 import ckan.plugins as p
 import ckan.plugins.toolkit as tk
 from ckan.exceptions import CkanConfigurationException
@@ -35,14 +37,14 @@ DEFAULT_FORM_DEFINITION = json.dumps(
                 {"value": "private", "label": "Private"},
             ],
         },
-    }
+    },
 )
 DEFAULT_FIELD_ORDER = None
 
 
 def form_config():
     definition = json.loads(
-        tk.config.get(CONFIG_FORM_DEFINITION, DEFAULT_FORM_DEFINITION)
+        tk.config.get(CONFIG_FORM_DEFINITION, DEFAULT_FORM_DEFINITION),
     )
     order = tk.aslist(tk.config.get(CONFIG_FIELD_ORDER, DEFAULT_FIELD_ORDER))
     if not order:
@@ -72,17 +74,18 @@ class AdvancedSearchPlugin(p.SingletonPlugin):
             from ckanext.composite_search.interfaces import ICompositeSearch
         except ImportError:
             raise CkanConfigurationException(
-                "ckanext-composite-search is not installed"
+                "ckanext-composite-search is not installed",
             )
         if not p.plugin_loaded("composite_search"):
-            raise CkanConfigurationException(
-                "`composite_search` plugin must be enabled in order to use advanced search"
-            )
+            msg = "Advanced search requires `composite_search` plugin"
+            raise CkanConfigurationException(msg)
         if not list(p.PluginImplementations(ICompositeSearch)):
-            raise CkanConfigurationException(
-                "Advanced search requires plugin that implements ICompositeSearch."
-                + " Consider enabling `default_composite_search` plugins."
+            msg = (
+                "Advanced search requires plugin that implements"
+                + " ICompositeSearch. Consider enabling "
+                + "`default_composite_search` plugins."
             )
+            raise CkanConfigurationException(msg)
 
     # ITemplateHelpers
 
@@ -93,9 +96,12 @@ class AdvancedSearchPlugin(p.SingletonPlugin):
 
     # IPackageController
 
-    def before_search(self, search_params: dict[str, Any]):
+    def before_dataset_search(self, search_params: dict[str, Any]):
         solr_q = search_params.get("extras", {}).get("ext_solr_q", None)
         if solr_q:
             search_params.setdefault("q", "")
             search_params["q"] += " " + solr_q
         return search_params
+
+    if not tk.check_ckan_version("2.10"):
+        before_search = before_dataset_search

{ckanext-search-tweaks-0.4.12 → ckanext-search-tweaks-0.6.0}/ckanext/search_tweaks/cli.py

@@ -1,8 +1,8 @@
 import click
 
-
-def get_commands():
-    return [search_tweaks]
+__all__ = [
+    "search_tweaks",
+]
 
 
 @click.group(short_help="Search tweaks")

ckanext-search-tweaks-0.6.0/ckanext/search_tweaks/config.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+import ckan.plugins.toolkit as tk
+from ckan.lib.search.query import QUERY_FIELDS
+
+CONFIG_QF = "ckanext.search_tweaks.common.qf"
+DEFAULT_QF = QUERY_FIELDS
+
+CONFIG_FUZZY = "ckanext.search_tweaks.common.fuzzy_search.enabled"
+CONFIG_FUZZY_DISTANCE = "ckanext.search_tweaks.common.fuzzy_search.distance"
+CONFIG_MM = "ckanext.search_tweaks.common.mm"
+CONFIG_FUZZY_KEEP_ORIGINAL = "ckanext.search_tweaks.common.fuzzy_search.keep_original"
+CONFIG_PREFER_BOOST = "ckanext.search_tweaks.common.prefer_boost"
+
+
+def qf() -> str:
+    return tk.config[CONFIG_QF] or DEFAULT_QF
+
+
+def fuzzy() -> bool:
+    return tk.config[CONFIG_FUZZY]
+
+
+def fuzzy_distance() -> int:
+    return tk.config[CONFIG_FUZZY_DISTANCE]
+
+
+def mm() -> str:
+    return tk.config[CONFIG_MM]
+
+
+def fuzzy_with_original() -> bool:
+    return tk.config[CONFIG_FUZZY_KEEP_ORIGINAL]
+
+
+def prefer_boost() -> bool:
+    return tk.config[CONFIG_PREFER_BOOST]

{ckanext-search-tweaks-0.4.12 → ckanext-search-tweaks-0.6.0}/ckanext/search_tweaks/field_relevance/plugin.py

@@ -1,13 +1,13 @@
 from __future__ import annotations
 
-from typing import Any, Optional
+from typing import Any
 
 import ckan.plugins as p
 import ckan.plugins.toolkit as tk
 
+from ckanext.search_tweaks import feature_disabled
+from ckanext.search_tweaks.interfaces import ISearchTweaks
 from . import views
-from ..interfaces import ISearchTweaks
-from .. import feature_disabled
 
 CONFIG_BOOST_FN = "ckanext.search_tweaks.field_relevance.boost_function"
 
@@ -21,11 +21,9 @@ class FieldRelevancePlugin(p.SingletonPlugin):
     p.implements(p.IConfigurer, inherit=True)
 
     # ISearchTweaks
-    def get_search_boost_fn(
-        self, search_params: dict[str, Any]
-    ) -> Optional[str]:
+    def get_search_boost_fn(self, search_params: dict[str, Any]) -> str | None:
         if feature_disabled("field_boost", search_params):
-            return
+            return None
 
         return tk.config.get(CONFIG_BOOST_FN, DEFAULT_BOOST_FN)
 
@@ -40,15 +38,14 @@ class FieldRelevancePlugin(p.SingletonPlugin):
         tk.add_template_directory(config, "templates")
         tk.add_resource("assets", "search_tweaks_field_relevance")
 
-
     # IAuthFunctions
     def get_auth_functions(self):
         return {
-            "search_tweaks_field_relevance_promote": search_tweaks_field_relevance_promote,
+            "search_tweaks_field_relevance_promote": promote_auth,
         }
 
 
-def search_tweaks_field_relevance_promote(context, data_dict):
+def promote_auth(context, data_dict):
     try:
         tk.check_access("package_update", context, data_dict)
     except tk.NotAuthorized:

{ckanext-search-tweaks-0.4.12 → ckanext-search-tweaks-0.6.0}/ckanext/search_tweaks/field_relevance/views.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
-from typing import Any, Optional
+
+from typing import Any
 
 from flask import Blueprint
 from flask.views import MethodView
@@ -10,9 +11,7 @@ import ckan.plugins.toolkit as tk
 CONFIG_ENABLE_PROMOTION_ROUTE = (
     "ckanext.search_tweaks.field_relevance.blueprint.promotion.enabled"
 )
-CONFIG_PROMOTION_PATH = (
-    "ckanext.search_tweaks.field_relevance.blueprint.promotion.path"
-)
+CONFIG_PROMOTION_PATH = "ckanext.search_tweaks.field_relevance.blueprint.promotion.path"
 CONFIG_MAX_PROMOTION = (
     "ckanext.search_tweaks.field_relevance.blueprint.promotion.max_value"
 )
@@ -30,14 +29,10 @@ field_relevance = Blueprint("search_tweaks_field_relevance", __name__)
 
 def get_blueprints():
     if tk.asbool(
-        tk.config.get(
-            CONFIG_ENABLE_PROMOTION_ROUTE, DEFAULT_ENABLE_PROMOTION_ROUTE
-        )
+        tk.config.get(CONFIG_ENABLE_PROMOTION_ROUTE, DEFAULT_ENABLE_PROMOTION_ROUTE),
     ):
         path = tk.config.get(CONFIG_PROMOTION_PATH, DEFAULT_PROMOTION_PATH)
-        field_relevance.add_url_rule(
-            path, view_func=PromoteView.as_view("promote")
-        )
+        field_relevance.add_url_rule(path, view_func=PromoteView.as_view("promote"))
 
     return [field_relevance]
 
@@ -57,9 +52,9 @@ class PromoteView(MethodView):
                 tk.get_validator("convert_int"),
                 tk.get_validator("natural_number_validator"),
                 tk.get_validator("limit_to_configured_maximum")(
-                    CONFIG_MAX_PROMOTION, DEFAULT_MAX_PROMOTION
+                    CONFIG_MAX_PROMOTION, DEFAULT_MAX_PROMOTION,
                 ),
-            ]
+            ],
         }
 
         data, errors = tk.navl_validate(
@@ -72,7 +67,7 @@ class PromoteView(MethodView):
             return self.get(id, data, errors)
         try:
             pkg_dict = tk.get_action("package_patch")(
-                {}, {"id": id, field: data[field]}
+                {}, {"id": id, field: data[field]},
             )
         except tk.ValidationError as e:
             for k, v in e.error_summary.items():
@@ -84,8 +79,8 @@ class PromoteView(MethodView):
     def get(
         self,
         id,
-        data: Optional[dict[str, Any]] = None,
-        errors: Optional[dict[str, Any]] = None,
+        data: dict[str, Any] | None = None,
+        errors: dict[str, Any] | None = None,
     ):
         self._check_access(id)
         field = tk.config.get(CONFIG_PROMOTION_FIELD, DEFAULT_PROMOTION_FIELD)
@@ -95,11 +90,9 @@ class PromoteView(MethodView):
             "errors": errors or {},
             "data": data or pkg_dict,
             "max_promotion": tk.asint(
-                tk.config.get(CONFIG_MAX_PROMOTION, DEFAULT_MAX_PROMOTION)
+                tk.config.get(CONFIG_MAX_PROMOTION, DEFAULT_MAX_PROMOTION),
             ),
             "field_name": field,
         }
 
-        return tk.render(
-            "search_tweaks/field_relevance/promote.html", extra_vars
-        )
+        return tk.render("search_tweaks/field_relevance/promote.html", extra_vars)

ckanext-search-tweaks-0.6.0/ckanext/search_tweaks/interfaces.py

@@ -0,0 +1,32 @@
+from __future__ import annotations
+
+from typing import Any
+
+from ckan.plugins.interfaces import Interface
+
+
+class ISearchTweaks(Interface):
+    def get_search_boost_fn(self, search_params: dict[str, Any]) -> str | None:
+        """Return Solr's boost function applicable to the current search.
+
+        Note: it will be applied as `boost` when
+        `ckanext.search_tweaks.common.prefer_boost` enabled and as `bf`
+        otherwise.
+
+        """
+        return None
+
+    def get_extra_qf(self, search_params: dict[str, Any]) -> str | None:
+        """Return an additional fragment of the Solr's qf.
+
+        This fragment will be appended to the current qf
+        """
+        return None
+
+
+
+class IQueryPopularity(Interface):
+    def skip_query_popularity(self, params: dict[str, Any]) -> bool:
+        """Do not index search query.
+        """
+        return False

{ckanext-search-tweaks-0.4.12 → ckanext-search-tweaks-0.6.0}/ckanext/search_tweaks/plugin.py

@@ -1,56 +1,37 @@
 from __future__ import annotations
 
 import logging
+from typing import Any
 
-from typing import Any, Dict
 import ckan.plugins as plugins
 import ckan.plugins.toolkit as tk
-from ckan.lib.search.query import QUERY_FIELDS
-
-from . import cli, boost_preffered, feature_disabled
+from . import feature_disabled, config
 from .interfaces import ISearchTweaks
 
 log = logging.getLogger(__name__)
-
-SearchParams = Dict[str, Any]
-
-CONFIG_QF = "ckanext.search_tweaks.common.qf"
-CONFIG_FUZZY = "ckanext.search_tweaks.common.fuzzy_search.enabled"
-CONFIG_FUZZY_DISTANCE = "ckanext.search_tweaks.common.fuzzy_search.distance"
-CONFIG_MM = "ckanext.search_tweaks.common.mm"
-CONFIG_FUZZY_KEEP_ORIGINAL = (
-    "ckanext.search_tweaks.common.fuzzy_search.keep_original"
-)
-
-DEFAULT_QF = QUERY_FIELDS
-DEFAULT_FUZZY = False
-DEFAULT_FUZZY_DISTANCE = 1
-DEFAULT_MM = "1"
-DEFAULT_FUZZY_KEEP_ORIGINAL = True
+CONFIG_PREFER_BOOST = "ckanext.search_tweaks.common.prefer_boost"
+DEFAULT_PREFER_BOOST = True
 
 
+@tk.blanket.cli
+@tk.blanket.config_declarations
 class SearchTweaksPlugin(plugins.SingletonPlugin):
-    plugins.implements(plugins.IClick)
     plugins.implements(plugins.IPackageController, inherit=True)
 
-    # IClick
-
-    def get_commands(self):
-        return cli.get_commands()
-
     # IPackageController
 
-    def before_dataset_search(self, search_params: SearchParams):
+    def before_dataset_search(self, search_params: dict[str, Any]):
         if feature_disabled("everything", search_params):
             return search_params
 
-        search_params.setdefault("mm", tk.config.get(CONFIG_MM, DEFAULT_MM))
+        search_params.setdefault("mm", config.mm())
 
         if "defType" not in search_params:
             search_params["defType"] = "edismax"
 
-        if boost_preffered() and search_params["defType"] == "edismax":
+        if config.prefer_boost() and search_params["defType"] == "edismax":
             _set_boost(search_params)
+
         else:
             _set_bf(search_params)
 
@@ -60,7 +41,7 @@ class SearchTweaksPlugin(plugins.SingletonPlugin):
         return search_params
 
 
-def _set_boost(search_params: SearchParams) -> None:
+def _set_boost(search_params: dict[str, Any]) -> None:
     boost: list[str] = search_params.setdefault("boost", [])
     for plugin in plugins.PluginImplementations(ISearchTweaks):
         extra = plugin.get_search_boost_fn(search_params)
@@ -69,7 +50,7 @@ def _set_boost(search_params: SearchParams) -> None:
         boost.append(extra)
 
 
-def _set_bf(search_params: SearchParams) -> None:
+def _set_bf(search_params: dict[str, Any]) -> None:
     default_bf: str = search_params.get("bf") or "0"
     search_params.setdefault("bf", default_bf)
     for plugin in plugins.PluginImplementations(ISearchTweaks):
@@ -79,13 +60,11 @@ def _set_bf(search_params: SearchParams) -> None:
         search_params["bf"] = f"sum({search_params['bf']},{extra_bf})"
 
 
-def _set_qf(search_params: SearchParams) -> None:
+def _set_qf(search_params: dict[str, Any]) -> None:
     if feature_disabled("qf", search_params):
         return
 
-    default_qf: str = search_params.get("qf") or tk.config.get(
-        CONFIG_QF, DEFAULT_QF
-    )
+    default_qf: str = search_params.get("qf") or config.qf()
     search_params.setdefault("qf", default_qf)
     for plugin in plugins.PluginImplementations(ISearchTweaks):
         extra_qf = plugin.get_extra_qf(search_params)
@@ -94,8 +73,8 @@ def _set_qf(search_params: SearchParams) -> None:
         search_params["qf"] += " " + extra_qf
 
 
-def _set_fuzzy(search_params: SearchParams) -> None:
-    if not tk.asbool(tk.config.get(CONFIG_FUZZY, DEFAULT_FUZZY)):
+def _set_fuzzy(search_params: dict[str, Any]) -> None:
+    if not config.fuzzy():
         return
 
     if feature_disabled("fuzzy", search_params):
@@ -118,20 +97,16 @@ def _set_fuzzy(search_params: SearchParams) -> None:
             if s.isalpha() and s not in ("AND", "OR", "TO")
             else s,
             q.split(),
-        )
+        ),
     )
-    if tk.asbool(
-        tk.config.get(CONFIG_FUZZY_KEEP_ORIGINAL, DEFAULT_FUZZY_KEEP_ORIGINAL)
-    ):
+    if config.fuzzy_with_original():
         search_params["q"] = f"({fuzzy_q}) OR ({q})"
     else:
         search_params["q"] = fuzzy_q
 
 
 def _get_fuzzy_distance() -> int:
-    distance = tk.asint(
-        tk.config.get(CONFIG_FUZZY_DISTANCE, DEFAULT_FUZZY_DISTANCE)
-    )
+    distance = config.fuzzy_distance()
     if distance < 0:
         log.warning("Cannot use negative fuzzy distance: %s.", distance)
         distance = 0

ckanext-search-tweaks-0.6.0/ckanext/search_tweaks/query_popularity/config.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+import ckan.plugins.toolkit as tk
+
+
+def skip_irrefutable() -> bool:
+    return tk.config["ckanext.search_tweaks.query_popularity.skip_irrefutable_search"]
+
+
+def ignored_symbols() -> set[str]:
+    return set(tk.config["ckanext.search_tweaks.query_popularity.ignored_symbols"])
+
+
+def ignored_terms() -> list[str]:
+    return tk.config["ckanext.search_tweaks.query_popularity.ignored_terms"]
+
+
+def throttle() -> int:
+    return tk.config["ckanext.search_tweaks.query_popularity.query_throttle"]
+
+
+def max_age() -> int:
+    return tk.config["ckanext.search_tweaks.query_popularity.max_age"]
+
+
+def obsoletion_period() -> int:
+    return tk.config["ckanext.search_tweaks.query_popularity.obsoletion_period"]
+
+
+def tracked_endpoints() -> list[str]:
+    return tk.config["ckanext.search_tweaks.query_popularity.tracked_endpoints"]

ckanext-search-tweaks-0.6.0/ckanext/search_tweaks/query_popularity/logic/action.py

@@ -0,0 +1,43 @@
+from __future__ import annotations
+from typing import Any
+from ckan import types
+import ckan.plugins.toolkit as tk
+
+from ckanext.search_tweaks.query_popularity.score import Score
+
+
+@tk.side_effect_free
+def search_tweaks_query_popularity_list(
+    context: types.Context, data_dict: dict[str, Any]
+) -> list[dict[str, Any]]:
+    score = Score()
+
+    if tk.asbool(data_dict.get("refresh")):
+        score.refresh()
+
+    limit = tk.asint(data_dict.get("limit", 10))
+
+    return list(score.stats(limit))
+
+
+@tk.side_effect_free
+def search_tweaks_query_popularity_export(
+    context: types.Context, data_dict: dict[str, Any]
+) -> dict[str, Any]:
+    score = Score()
+
+    results = score.export()
+    return {"results": results, "count": len(results)}
+
+
+@tk.side_effect_free
+def search_tweaks_query_popularity_ignore(
+    context: types.Context, data_dict: dict[str, Any]
+):
+    q = tk.get_or_bust(data_dict, "q")
+    score = Score()
+    result = score.ignore(q)
+    if tk.asbool(data_dict.get("remove")):
+        score.drop(q)
+
+    return result

ckanext-search-tweaks-0.6.0/ckanext/search_tweaks/query_popularity/logic/auth.py

@@ -0,0 +1,23 @@
+from __future__ import annotations
+from typing import Any
+from ckan import types
+
+from ckan.authz import is_authorized
+
+
+def search_tweaks_query_popularity_list(
+    context: types.Context, data_dict: dict[str, Any]
+) -> types.AuthResult:
+    return is_authorized("sysadmin", context, data_dict)
+
+
+def search_tweaks_query_popularity_export(
+    context: types.Context, data_dict: dict[str, Any]
+) -> types.AuthResult:
+    return is_authorized("sysadmin", context, data_dict)
+
+
+def search_tweaks_query_popularity_ignore(
+    context: types.Context, data_dict: dict[str, Any]
+) -> types.AuthResult:
+    return is_authorized("sysadmin", context, data_dict)

ckanext-search-tweaks-0.6.0/ckanext/search_tweaks/query_popularity/plugin.py

@@ -0,0 +1,47 @@
+from __future__ import annotations
+from typing import Any
+import ckan.plugins as p
+import ckan.plugins.toolkit as tk
+from ckanext.search_tweaks.interfaces import IQueryPopularity
+from . import config, score
+
+
+@tk.blanket.actions
+@tk.blanket.auth_functions
+@tk.blanket.config_declarations
+class QueryPopularityPlugin(p.SingletonPlugin):
+    p.implements(p.IConfigurable)
+    p.implements(p.IPackageController, inherit=True)
+    p.implements(IQueryPopularity, inherit=True)
+
+    def after_dataset_search(self, results: dict[str, Any], params: dict[str, Any]):
+        bp, view = tk.get_endpoint()
+        if bp and view and f"{bp}.{view}" in config.tracked_endpoints():
+            if not any(
+                plugin.skip_query_popularity(params)
+                for plugin in p.PluginImplementations(IQueryPopularity)
+            ):
+                self.score.save(params["q"])
+
+        return results
+
+    def configure(self, config: Any):
+        self.score = score.Score()
+
+    def skip_query_popularity(self, params: dict[str, Any]) -> bool:
+        q = params["q"]
+
+        if q == "*:*":
+            return config.skip_irrefutable()
+
+        symbols = config.ignored_symbols()
+        if symbols and set(q) & symbols:
+            return True
+
+        terms = config.ignored_terms()
+
+        for term in terms:
+            if term in q:
+                return True
+
+        return False