ckanext-search-tweaks 0.6.2__py3-none-any.whl → 1.0.0__py3-none-any.whl

ckanext/__init__.py

@@ -1,4 +1,3 @@
-
 # this is a namespace package
 try:
     import pkg_resources

ckanext/search_tweaks/advanced_search/assets/advanced-search.css

@@ -24,7 +24,3 @@
 .search-tweaks-advanced-search.enabled.use-solr-query .advanced-search-submit {
   display: none;
 }
-
-.advanced-toggles label:after {
-  content: none;
-}

ckanext/search_tweaks/advanced_search/plugin.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 import json
 from typing import Any
 
+from ckan import types
 import ckan.plugins as p
 import ckan.plugins.toolkit as tk
 from ckan.exceptions import CkanConfigurationException
@@ -55,6 +56,16 @@ def form_config():
     }
 
 
+@tk.side_effect_free
+def advanced_search_config(
+    context: types.Context,
+    data_dict: dict[str, Any],
+) -> dict[str, Any]:
+    """Configuration for advanced search fields."""
+    return tk.h.advanced_search_form_config()
+
+
+@tk.blanket.actions({"advanced_search_config": advanced_search_config})
 class AdvancedSearchPlugin(p.SingletonPlugin):
     p.implements(p.IConfigurer, inherit=True)
     p.implements(p.IConfigurable)

ckanext/search_tweaks/advanced_search/templates/advanced_search/search_form.html

@@ -39,7 +39,8 @@ solr_syntax_blank - open SOLR syntax article in the new tab
 <div class="input-group search-input-group">
     <input type="text" class="form-control input-lg"
 	   name="q" value="{{ query }}" autocomplete="off"
-	   placeholder="{{ placeholder }}">
+	   placeholder="{{ placeholder }}"
+	   aria-label="Search data">
     {{ search_button }}
 </div>
 {% endset %}
@@ -50,7 +51,8 @@ solr_syntax_blank - open SOLR syntax article in the new tab
     <input type="text" class="form-control input-lg"
 	   value="{{ request.args.ext_solr_q }}" autocomplete="off"
 	   name="ext_solr_q" disabled
-	   placeholder="{{ labels.solr_placeholder or _('metadata_created:[NOW/YEAR TO *] -tags:Health') }}" >
+	   placeholder="{{ labels.solr_placeholder or _('metadata_created:[NOW/YEAR TO *] -tags:Health') }}"
+	   aria-label="Search using Solr query">
     {{ search_button }}
 </div>
 {% endset %}
@@ -95,7 +97,7 @@ solr_syntax_blank - open SOLR syntax article in the new tab
 					{{ labels.solr_toggle or _('Add query syntax to search') }}
 					</label>
 					{% block solr_syntax_url %}
-						<a {% if solr_syntax_blank %}target="_blank"{% endif %} href="{{ solr_syntax_url }}">
+						<a {% if solr_syntax_blank %}target="_blank"{% endif %} href="{{ solr_syntax_url }}" aria-label="Learn more about SOLR query parameters">
 						<i class="fa fa-info-circle" data-placement="top" data-toggle="tooltip"
 						title="{{ labels.solr_explanation or _('This adds SOLR query language, for more information on how to use click here') }}"></i>
 						</a>

ckanext/search_tweaks/config_declaration.yaml

@@ -0,0 +1,24 @@
+version: 1
+groups:
+  - annotation: "ckanext-search-tweaks: common"
+    options:
+      - key: ckanext.search_tweaks.common.qf
+        placeholder: "name^4 title^4 tags^2 groups^2 text"
+
+      - key: ckanext.search_tweaks.common.mm
+        default: "1"
+
+      - key: ckanext.search_tweaks.common.fuzzy_search.enabled
+        type: bool
+
+      - key: ckanext.search_tweaks.common.fuzzy_search.distance
+        type: int
+        default: 1
+
+      - key: ckanext.search_tweaks.common.fuzzy_search.keep_original
+        type: bool
+        default: true
+
+      - key: ckanext.search_tweaks.common.prefer_boost
+        type: bool
+        default: true

ckanext/search_tweaks/field_relevance/plugin.py

@@ -8,8 +8,6 @@ import ckan.plugins.toolkit as tk
 from ckanext.search_tweaks.interfaces import ISearchTweaks
 from ckanext.search_tweaks.shared import feature_disabled
 
-from . import views
-
 CONFIG_BOOST_FN = "ckanext.search_tweaks.field_relevance.boost_function"
 
 DEFAULT_BOOST_FN = None

ckanext/search_tweaks/field_relevance/templates/search_tweaks/field_relevance/promote.html

@@ -17,7 +17,7 @@
 	<form id="dataset-edit" class="dataset-form" method="post" action="{{ action }}" data-module="basic-form" novalidate>
 
 	    {% call form.input_block(field_name, _("Promotion level"), errors[field_name], classes=["search-tweaks-label-with-value"]) %}
-		<input data-module="search-tweaks-reflect-range-in-label" data-module-format=" (%d)" id="{{ field_name }}" type="range" name="{{ field_name }}" value="{{ data[field_name] }}" max="{{ max_promotion }}" />
+		<input data-module="search-tweaks-reflect-range-in-label" data-module-format=" (%d)" id="{{ field_name }}" type="range" name="{{ field_name }}" value="{{ data[field_name] }}" min="{{ min_promotion }}" max="{{ max_promotion }}" />
 	    {% endcall %}
 
 	    {% block form_actions %}

ckanext/search_tweaks/field_relevance/views.py

@@ -5,6 +5,7 @@ from typing import Any
 from flask import Blueprint
 from flask.views import MethodView
 
+import ckan.types as types
 import ckan.model as model
 import ckan.plugins.toolkit as tk
 
@@ -12,6 +13,9 @@ CONFIG_ENABLE_PROMOTION_ROUTE = (
     "ckanext.search_tweaks.field_relevance.blueprint.promotion.enabled"
 )
 CONFIG_PROMOTION_PATH = "ckanext.search_tweaks.field_relevance.blueprint.promotion.path"
+CONFIG_MIN_PROMOTION = (
+    "ckanext.search_tweaks.field_relevance.blueprint.promotion.min_value"
+)
 CONFIG_MAX_PROMOTION = (
     "ckanext.search_tweaks.field_relevance.blueprint.promotion.max_value"
 )
@@ -21,6 +25,7 @@ CONFIG_PROMOTION_FIELD = (
 
 DEFAULT_ENABLE_PROMOTION_ROUTE = False
 DEFAULT_PROMOTION_PATH = "/dataset/promote/<id>"
+DEFAULT_MIN_PROMOTION = 0
 DEFAULT_MAX_PROMOTION = 100
 DEFAULT_PROMOTION_FIELD = "promotion_level"
 
@@ -40,9 +45,10 @@ class PromoteView(MethodView):
         schema = {
             field: [
                 tk.get_validator("convert_int"),
-                tk.get_validator("natural_number_validator"),
+                tk.get_validator("int_validator"),
                 tk.get_validator("limit_to_configured_maximum")(
-                    CONFIG_MAX_PROMOTION, DEFAULT_MAX_PROMOTION,
+                    CONFIG_MAX_PROMOTION,
+                    DEFAULT_MAX_PROMOTION,
                 ),
             ],
         }
@@ -50,14 +56,15 @@ class PromoteView(MethodView):
         data, errors = tk.navl_validate(
             dict(tk.request.form),
             schema,
-            {"model": model, "session": model.Session},
+            types.Context(model=model, session=model.Session),  # type: ignore
         )
 
         if errors:
             return self.get(id, data, errors)
         try:
             pkg_dict = tk.get_action("package_patch")(
-                {}, {"id": id, field: data[field]},
+                {},
+                {"id": id, field: data[field]},
             )
         except tk.ValidationError as e:
             for k, v in e.error_summary.items():
@@ -79,6 +86,9 @@ class PromoteView(MethodView):
             "pkg_dict": pkg_dict,
             "errors": errors or {},
             "data": data or pkg_dict,
+            "min_promotion": tk.asint(
+                tk.config.get(CONFIG_MIN_PROMOTION, DEFAULT_MIN_PROMOTION),
+            ),
             "max_promotion": tk.asint(
                 tk.config.get(CONFIG_MAX_PROMOTION, DEFAULT_MAX_PROMOTION),
             ),
@@ -91,4 +101,7 @@ class PromoteView(MethodView):
 if tk.asbool(
     tk.config.get(CONFIG_ENABLE_PROMOTION_ROUTE, DEFAULT_ENABLE_PROMOTION_ROUTE),
 ):
-    field_relevance.add_url_rule("/dataset/promote/<id>", view_func=PromoteView.as_view("promote"))
+    field_relevance.add_url_rule(
+        "/dataset/promote/<id>",
+        view_func=PromoteView.as_view("promote"),
+    )

ckanext/search_tweaks/interfaces.py

@@ -24,9 +24,7 @@ class ISearchTweaks(Interface):
         return None
 
 
-
 class IQueryPopularity(Interface):
     def skip_query_popularity(self, params: dict[str, Any]) -> bool:
-        """Do not index search query.
-        """
+        """Do not index search query."""
         return False

ckanext/search_tweaks/plugin.py

@@ -33,7 +33,6 @@ class SearchTweaksPlugin(plugins.SingletonPlugin):
 
         if config.prefer_boost() and search_params["defType"] == "edismax":
             _set_boost(search_params)
-
         else:
             _set_bf(search_params)
 

ckanext/search_tweaks/query_popularity/config_declaration.yaml

@@ -0,0 +1,29 @@
+version: 1
+groups:
+  - annotation: "ckanext-search-tweaks: query popularity"
+    options:
+      - key: ckanext.search_tweaks.query_popularity.skip_irrefutable_search
+        type: bool
+        default: true
+
+      - key: ckanext.search_tweaks.query_popularity.ignored_symbols
+        default: ":"
+
+      - key: ckanext.search_tweaks.query_popularity.ignored_terms
+        type: list
+
+      - key: ckanext.search_tweaks.query_popularity.tracked_endpoints
+        type: list
+        default: dataset.search
+
+      - key: ckanext.search_tweaks.query_popularity.query_throttle
+        type: int
+        default: 86_400 # 24 hours
+
+      - key: ckanext.search_tweaks.query_popularity.max_age
+        type: int
+        default: 2_592_000 # 30 days
+
+      - key: ckanext.search_tweaks.query_popularity.obsoletion_period
+        type: int
+        default: 86_400 # 24 hours

ckanext/search_tweaks/query_popularity/logic/schema.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 from ckan.logic.schema import validator_args
 
+
 @validator_args
 def query_popularity_import(not_empty, boolean_validator, convert_to_json_if_string):
     return {

ckanext/search_tweaks/query_popularity/score.py

@@ -1,7 +1,8 @@
 from __future__ import annotations
 
 import logging
-from typing import Any, Iterable
+from typing import Any
+from collections.abc import Iterable
 
 from ckanext.toolbelt.utils.tracking import DateTracker
 

ckanext/search_tweaks/query_relevance/__init__.py

@@ -13,38 +13,63 @@ from .score import QueryScore, normalize_query
 __all__ = ["QueryScore", "normalize_query", "update_score_by_url"]
 
 
-def update_score_by_url(pkg: model.Package, ref: str | None = None) -> bool:
-    """Make given package more relevant for the current search query."""
-    if tk.request:
-        ref = ref or tk.request.referrer
+def update_score_by_url(pkg: model.Package, referrer: str | None = None) -> bool:
+    """Boost the relevance of the given package for the current search query
 
-    if not ref:
+    Args:
+        pkg: the package to boost
+        referrer: the URL of the current request
+
+    Returns:
+        True if the package was boosted, False otherwise
+    """
+
+    referrer = referrer or (tk.request.referrer if tk.request else None)
+
+    if not referrer:
         return False
 
-    url = urlparse(ref)
-    if not _path_has_score_for(url.path, pkg):
+    url = urlparse(referrer)
+
+    if not _is_scoring_enabled_for_path(url.path, pkg):
         return False
 
     query = parse_qs(url.query.lstrip("?"))
+
     if "q" not in query:
         return False
-    q = query["q"][0]
 
-    score = QueryScore(pkg.id, q)
-    score.increase(1)
+    QueryScore(pkg.id, query["q"][0]).increase(1)
+
     return True
 
 
-def _path_has_score_for(path: str, pkg: model.Package) -> bool:
+def _is_scoring_enabled_for_path(path: str, package: model.Package) -> bool:
+    """
+    Determine if a given URL path should have scoring enabled.
+
+    Checks if the provided path matches any of the following URL patterns that
+    support scoring functionality:
+
+    Args:
+        path: The URL path to check
+        package: The package object containing type and owner_org info
+
+    Returns:
+        True if the path should have scoring enabled, False otherwise
+    """
+
     path = path.rstrip("/")
+
     if path == tk.h.url_for("dataset.search").rstrip("/"):
         return True
 
     with contextlib.suppress(BuildError):
-        if path == tk.h.url_for(pkg.type + ".search").rstrip("/"):
+        if path == tk.h.url_for(package.type + ".search").rstrip("/"):
             return True
 
-    org = model.Group.get(pkg.owner_org)
+    org = model.Group.get(package.owner_org)
+
     if not org:
         return False
 

ckanext/search_tweaks/query_relevance/boost.py

@@ -0,0 +1,75 @@
+from __future__ import annotations
+
+from ckanext.search_tweaks.config import prefer_boost
+
+from . import QueryScore
+from .config import get_min_boost, get_max_boost, get_max_boost_count
+
+
+def build_boost_query_function(search_query: str) -> str | None:
+    """Build boost query function for given search query.
+
+    Args:
+        search_query: normalized query
+
+    Returns:
+        Boost function
+    """
+    boosts, max_score = get_boost_values(search_query)
+    min_boost = get_min_boost()
+    max_boost = get_max_boost()
+
+    if prefer_boost():
+        boost_expr = "1"
+
+        for pkg_id, raw_score in sorted(boosts.items(), reverse=True):
+            scaled = scale_score(raw_score, max_score, min_boost, max_boost)
+            boost_expr = f'if(eq(id,"{pkg_id}"),{scaled},{boost_expr})'
+
+        return f"sum(0, {boost_expr})"
+    else:
+        boost_parts = []
+
+        for pkg_id, raw_score in boosts.items():
+            score = scale_score(raw_score, max_score, min_boost, max_boost)
+
+            boost_parts.append(f'if(eq(id,"{pkg_id}"),{score},0)')
+
+        return f"sum(1,{','.join(boost_parts)})"
+
+
+def get_boost_values(search_query: str) -> tuple[dict[str, float], float]:
+    boosts = {}
+    max_score = 0
+
+    for entry in QueryScore.get_for_query(search_query):
+        package_id, score = entry
+
+        if score > max_score:
+            max_score = score
+
+        boosts[package_id.decode("utf-8")] = score
+
+    return boosts, max_score
+
+
+def scale_score(
+    value: float,
+    max_value: float,
+    min_boost: float,
+    max_boost: float,
+) -> float:
+    """
+    Linearly scales a value to the range [min_boost, max_boost].
+
+    This prevents datasets with high scores
+    from overpowering search relevance, ensuring more balanced results.
+    """
+    if max_value == 0:
+        return min_boost
+
+    value = max(0, min(value, max_value))
+
+    scaled = min_boost + (value / max_value) * (max_boost - min_boost)
+
+    return round(scaled, 4)

ckanext/search_tweaks/query_relevance/cli.py

@@ -1,15 +1,19 @@
+from __future__ import annotations
+
 import csv
 import datetime
+import logging
+from typing import TextIO
 
 import click
 import freezegun
 
 import ckan.model as model
-from ckan.lib.redis import connect_to_redis
 
 from . import QueryScore
 
-_search_csv_headers = ["package_id", "search_query", "count_of_hits"]
+
+log = logging.getLogger(__name__)
 
 
 @click.group(short_help="Manage search relevance")
@@ -20,69 +24,47 @@ def query():
 @query.command("import")
 @click.argument("source", type=click.File())
 @click.option("--date", type=datetime.date.fromisoformat)
-def import_source(source, date):
+def import_source(source: TextIO, date) -> None:
     """Import search stats from source"""
     if not date:
         date = datetime.date.today()
+
     with freezegun.freeze_time(date):
         reader = csv.DictReader(source)
         for row in reader:
             pkg = model.Package.get(row["package_id"])
+
             if not pkg:
                 click.secho(f"Package {row['package_id']} does not exists", fg="red")
                 continue
+
             score = QueryScore(pkg.id, row["search_query"])
             score.reset()
             score.increase(int(row["count_of_hits"]))
+
     click.secho("Done", fg="green")
 
 
 @query.command()
 @click.argument("output", type=click.File("w"), required=False)
-def export(output):
+def export(output: TextIO | None) -> None:
     """Export search stats into specified file."""
     rows = QueryScore.get_all()
+
     if output:
         writer = csv.writer(output)
-        writer.writerow(_search_csv_headers)
+        writer.writerow(["package_id", "search_query", "count_of_hits"])
         writer.writerows(rows)
     else:
         for row in rows:
-            click.echo("Id: %s, query: %s, count: %d" % row)
+            click.echo("ID: {}, query: {}, count: {}".format(*row))
+
     click.secho("Done", fg="green")
 
 
 @query.command()
-def align():
-    """Remove old records."""
-    rows = QueryScore.get_all()
-    for id_, query, _ in rows:
-        score = QueryScore(id_, query)
-        score.align()
-
+def reset() -> None:
+    """Reset query relevance scores"""
+    QueryScore.reset_all()
 
-@query.command()
-@click.option("--days", "-d", type=int, default=1)
-@click.argument("file")
-@click.pass_context
-def safe_export(ctx, days, file):
-    """Export stats if redis haven't been reloaded recently.
-
-    If redis runs less than N days, it was reloaded recently and contains no
-    stats. We have to import old snapshot into it.
-
-    If redis is up for N days and more, it contains relevant stats. We can
-    safely export them and overwrite old snapshot.
-
-    """
-    conn = connect_to_redis()
-    uptime = conn.info()["uptime_in_days"]
-    if uptime >= days:
-        click.secho(f"Redis runs for {uptime} days. Creating snapshot..", fg="green")
-        ctx.invoke(export, output=click.File("w")(file))
-    else:
-        click.secho(
-            f"Redis runs for {uptime} days. Restore stats from snapshot..",
-            fg="red",
-        )
-        ctx.invoke(import_source, source=click.File()(file))
+    click.secho("Done", fg="green")

ckanext/search_tweaks/query_relevance/config.py

@@ -0,0 +1,29 @@
+import ckan.plugins.toolkit as tk
+
+CONF_MIN_BOOST = "ckanext.search_tweaks.query_relevance.min_boost"
+CONF_MAX_BOOST = "ckanext.search_tweaks.query_relevance.max_boost"
+CONF_MAX_BOOST_COUNT = "ckanext.search_tweaks.query_relevance.max_boost_count"
+
+
+def get_min_boost() -> float:
+    return as_float(tk.config[CONF_MIN_BOOST])
+
+
+def get_max_boost() -> float:
+    return as_float(tk.config[CONF_MAX_BOOST])
+
+
+def get_max_boost_count() -> int:
+    return tk.config[CONF_MAX_BOOST_COUNT]
+
+
+def as_float(number: str) -> float:
+    """Convert a string into a float.
+
+    Example:
+        assert as_float("1.5") == 1.5
+    """
+    try:
+        return float(number)
+    except (TypeError, ValueError):
+        raise ValueError("Bad float value: {}".format(number))

ckanext/search_tweaks/query_relevance/config_declaration.yaml

@@ -0,0 +1,16 @@
+version: 1
+groups:
+  - annotation: "ckanext-search-tweaks:query_relevance"
+    options:
+      - key: ckanext.search_tweaks.query_relevance.min_boost
+        default: 1
+        description: Minimum boost to apply to a query. Use float values
+
+      - key: ckanext.search_tweaks.query_relevance.max_boost
+        default: 1.5
+        description: Maximum boost to apply to a query. Use float values
+
+      - key: ckanext.search_tweaks.query_relevance.max_boost_count
+        type: int
+        default: 60
+        description: Maximum number of boosts to apply to a query

ckanext/search_tweaks/query_relevance/plugin.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-from string import Template
 from typing import Any
 
 import ckan.plugins as plugins
@@ -10,15 +9,11 @@ from ckanext.search_tweaks.cli import attach_relevance_command
 from ckanext.search_tweaks.interfaces import ISearchTweaks
 from ckanext.search_tweaks.shared import feature_disabled
 
-from . import QueryScore, cli, normalize_query, update_score_by_url
-
-CONFIG_BOOST_STRING = "ckanext.search_tweaks.query_relevance.boost_function"
-CONFIG_RELEVANCE_PREFIX = "ckanext.search_tweaks.query_relevance.field_prefix"
-
-DEFAULT_BOOST_STRING = "scale(def($field,0),1,1.2)"
-DEFAULT_RELEVANCE_PREFIX = "query_relevance_"
+from . import cli, normalize_query, update_score_by_url
+from .boost import build_boost_query_function
 
 
+@tk.blanket.config_declarations
 class QueryRelevancePlugin(plugins.SingletonPlugin):
     plugins.implements(plugins.IConfigurable)
     plugins.implements(plugins.IPackageController, inherit=True)
@@ -31,15 +26,6 @@ class QueryRelevancePlugin(plugins.SingletonPlugin):
 
     # IPackageController
 
-    def before_dataset_index(self, pkg_dict):
-        prefix = tk.config.get(CONFIG_RELEVANCE_PREFIX, DEFAULT_RELEVANCE_PREFIX)
-
-        for _, query, score in QueryScore.get_for(pkg_dict["id"]):
-            query = query.replace(" ", "_")
-            pkg_dict[prefix + query] = score
-
-        return pkg_dict
-
     def read(self, entity):
         # update search relevance only for WEB-requests. Any kind of
         # CLI/search-index manipulations has no effect on it
@@ -49,27 +35,12 @@ class QueryRelevancePlugin(plugins.SingletonPlugin):
     # ISearchTweaks
 
     def get_search_boost_fn(self, search_params: dict[str, Any]) -> str | None:
-        if feature_disabled("query_boost", search_params):
-            return None
-
-        prefix = tk.config.get(CONFIG_RELEVANCE_PREFIX, DEFAULT_RELEVANCE_PREFIX)
-        disabled = tk.asbool(
-            search_params.get("extras", {}).get(
-                "ext_search_tweaks_disable_relevance",
-                False,
-            ),
-        )
-
-        if not search_params.get("q") or disabled:
-            return None
+        q = search_params.get("q")
 
-        normalized = normalize_query(search_params["q"]).replace(" ", "_")
-        if not normalized:
+        if feature_disabled("query_boost", search_params) or not q:
             return None
 
-        field = prefix + normalized
-        boost_string = Template(
-            tk.config.get(CONFIG_BOOST_STRING, DEFAULT_BOOST_STRING),
-        )
+        if normalized := normalize_query(q).replace(" ", "_"):
+            return build_boost_query_function(normalized)
 
-        return boost_string.safe_substitute({"field": field})
+        return None