rer.solrpush 1.4.2__tar.gz → 1.5.0__tar.gz

{rer.solrpush-1.4.2 → rer.solrpush-1.5.0}/CHANGES.rst

@@ -2,6 +2,25 @@ Changelog
 =========
 
 
+1.5.0 (2024-12-05)
+------------------
+
+- Remove new line characters in get_site_title method.
+  [cekk]
+- Change elevate schema to be editable with Volto.
+  [cekk]
+- Partly remove Python2 compatibility.
+  [cekk]
+- Fix pagination in querybuilder for solr.
+  [cekk]
+
+
+1.4.3 (2024-10-11)
+------------------
+
+- Fix SolrIndexProcessor logic: avoid not needed reindexes when we are reindexing objects with indexes not in SOLR.
+  [cekk]
+
 1.4.2 (2024-08-08)
 ------------------
 

{rer.solrpush-1.4.2 → rer.solrpush-1.5.0}/MANIFEST.in

@@ -24,3 +24,9 @@ exclude requirements.txt
 exclude tox.ini
 exclude webpack.config.js
 exclude yarn.lock
+exclude .coveragerc
+exclude base.cfg
+exclude bobtemplate.cfg
+exclude buildout.cfg
+exclude test_plone52.cfg
+exclude test_plone60.cfg

{rer.solrpush-1.4.2 → rer.solrpush-1.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: rer.solrpush
-Version: 1.4.2
+Version: 1.5.0
 Summary: Prodotto per Regione Emilia-Romagna relativo all'indicizzazione dei contenuti con solr
 Home-page: https://github.com/collective/rer.solrpush
 Author: RedTurtle
@@ -244,6 +244,25 @@ Changelog
 =========
 
 
+1.5.0 (2024-12-05)
+------------------
+
+- Remove new line characters in get_site_title method.
+  [cekk]
+- Change elevate schema to be editable with Volto.
+  [cekk]
+- Partly remove Python2 compatibility.
+  [cekk]
+- Fix pagination in querybuilder for solr.
+  [cekk]
+
+
+1.4.3 (2024-10-11)
+------------------
+
+- Fix SolrIndexProcessor logic: avoid not needed reindexes when we are reindexing objects with indexes not in SOLR.
+  [cekk]
+
 1.4.2 (2024-08-08)
 ------------------
 

{rer.solrpush-1.4.2 → rer.solrpush-1.5.0}/setup.cfg

@@ -6,11 +6,7 @@ ignore =
 	.gitattributes
 
 [isort]
-force_alphabetical_sort = True
-force_single_line = True
-lines_after_imports = 2
-line_length = 200
-not_skip = __init__.py
+profile = plone
 
 [flake8]
 ignore = 
@@ -19,7 +15,7 @@ ignore =
 	E501
 	T001
 	C813
-	C901
+	C101
 exclude = bootstrap.py,docs,*.egg.,omelette
 max-line-length = 88
 max-complexity = 18

{rer.solrpush-1.4.2 → rer.solrpush-1.5.0}/setup.py

@@ -16,7 +16,7 @@ long_description = "\n\n".join(
 
 setup(
     name="rer.solrpush",
-    version="1.4.2",
+    version="1.5.0",
     description="Prodotto per Regione Emilia-Romagna relativo all'indicizzazione dei contenuti con solr",
     long_description=long_description,
     # Get more from https://pypi.python.org/pypi?%3Aaction=list_classifiers
@@ -59,7 +59,7 @@ setup(
         "plone.api>=1.8.4",
         "pysolr",
         'ftfy==4.4.3;python_version<="2.7"',
-        "plone.restapi",
+        "plone.restapi>=8.13.0",
         "z3c.jbot",
     ],
     extras_require={

{rer.solrpush-1.4.2 → rer.solrpush-1.5.0}/src/rer/solrpush/adapters/configure.zcml

@@ -12,4 +12,10 @@
   <!-- RSS feed -->
   <adapter factory=".rss_feed.SOLRFeedItem" />
 
+  <!-- Summary Serializer Metadata -->
+  <utility
+      factory=".summary.JSONSummarySerializerMetadata"
+      name="rer.solrpush.summary_serializer_metadata"
+      />
+
 </configure>

rer.solrpush-1.5.0/src/rer/solrpush/adapters/summary.py

@@ -0,0 +1,11 @@
+from plone.restapi.interfaces import IJSONSummarySerializerMetadata
+from zope.interface import implementer
+
+
+@implementer(IJSONSummarySerializerMetadata)
+class JSONSummarySerializerMetadata:
+    def default_metadata_fields(self):
+        """
+        Force always return some metadata
+        """
+        return {"[elevated]", "Date"}

{rer.solrpush-1.4.2 → rer.solrpush-1.5.0}/src/rer/solrpush/behaviors/solr_fields.py

@@ -11,9 +11,7 @@ from zope.interface import provider
 class ISolrFields(model.Schema):
     """Additional fields to control Solr integration"""
 
-    directives.fieldset(
-        "categorization", fields=["showinsearch", "searchwords"]
-    )
+    directives.fieldset("categorization", fields=["showinsearch", "searchwords"])
 
     showinsearch = schema.Bool(
         required=False,

{rer.solrpush-1.4.2 → rer.solrpush-1.5.0}/src/rer/solrpush/browser/controlpanel.py

@@ -63,9 +63,7 @@ class RerSolrpushEditForm(RegistryEditForm):
         api.portal.show_message(_("Changes saved."), request=self.request)
         init_error = init_solr_push()
         if init_error:
-            api.portal.show_message(
-                init_error, type="error", request=self.request
-            )
+            api.portal.show_message(init_error, type="error", request=self.request)
         else:
             api.portal.show_message(
                 _("Loaded schema.xml from SOLR"), request=self.request
@@ -82,9 +80,7 @@ class RerSolrpushEditForm(RegistryEditForm):
 
         init_error = init_solr_push()
         if init_error:
-            api.portal.show_message(
-                init_error, type="error", request=self.request
-            )
+            api.portal.show_message(init_error, type="error", request=self.request)
         else:
             api.portal.show_message(
                 _("Reloaded schema.xml from SOLR"), request=self.request

{rer.solrpush-1.4.2 → rer.solrpush-1.5.0}/src/rer/solrpush/browser/elevate_settings.py

@@ -1,11 +1,8 @@
 # -*- coding: utf-8 -*-
-from collective.z3cform.jsonwidget.browser.widget import JSONFieldWidget
 from plone.app.registry.browser.controlpanel import ControlPanelFormWrapper
 from plone.app.registry.browser.controlpanel import RegistryEditForm
-from Products.CMFPlone.resources import add_bundle_on_request
 from rer.solrpush import _
 from rer.solrpush.interfaces import IElevateSettings
-from rer.solrpush.interfaces.elevate import IElevateRowSchema
 from z3c.form import field
 
 
@@ -17,17 +14,7 @@ class ElevateSettingsEditForm(RegistryEditForm):
     )
 
     fields = field.Fields(IElevateSettings)
-    fields["elevate_schema"].widgetFactory = JSONFieldWidget
-
-    def updateWidgets(self):
-        """ """
-        super(ElevateSettingsEditForm, self).updateWidgets()
-        self.widgets["elevate_schema"].schema = IElevateRowSchema
 
 
 class ElevateSettingsView(ControlPanelFormWrapper):
     form = ElevateSettingsEditForm
-
-    def __call__(self):
-        add_bundle_on_request(self.request, "z3cform-jsonwidget-bundle")
-        return super(ElevateSettingsView, self).__call__()

{rer.solrpush-1.4.2 → rer.solrpush-1.5.0}/src/rer/solrpush/browser/maintenance.py

@@ -14,8 +14,8 @@ from rer.solrpush.utils import remove_from_solr
 from rer.solrpush.utils import reset_solr
 from rer.solrpush.utils import search
 from rer.solrpush.utils.solr_common import is_solr_active
+from rer.solrpush.utils.solr_indexer import all_site_titles
 from rer.solrpush.utils.solr_indexer import can_index
-from rer.solrpush.utils.solr_indexer import get_site_title
 from rer.solrpush.utils.solr_indexer import parse_date_as_datetime
 from time import strftime
 from time import time
@@ -33,9 +33,7 @@ import os
 import pkg_resources
 
 
-JS_TEMPLATE = (
-    "{portal_url}/++plone++rer.solrpush/dist/{env_mode}/{name}.js?v={version}"
-)
+JS_TEMPLATE = "{portal_url}/++plone++rer.solrpush/dist/{env_mode}/{name}.js?v={version}"
 CSS_TEMPLATE = (
     "{portal_url}/++plone++rer.solrpush/dist/{env_mode}/{name}.css?v={version}"
 )
@@ -90,10 +88,9 @@ class ResetSolr(SolrMaintenanceBaseForm):
     )
 
     def do_action(self):
+        self.request._rest_cors_preflight = True
         reset_solr()
-        msg_label = _(
-            "maintenance_reset_success", default="SOLR index dropped"
-        )
+        msg_label = _("maintenance_reset_success", default="SOLR index dropped")
         logger.info("##### SOLR Index dropped #####")
         api.portal.show_message(message=msg_label, request=self.request)
         return self.request.response.redirect(
@@ -137,9 +134,7 @@ class ReindexBaseView(BrowserView):
         if not self.solr_utility:
             return
         if not is_solr_active():
-            logger.warning(
-                "Trying to reindexing but solr is not set as active"
-            )
+            logger.warning("Trying to reindexing but solr is not set as active")
             return
         elapsed = timer()
         if self.solr_utility.enabled_types:
@@ -155,9 +150,7 @@ class ReindexBaseView(BrowserView):
         )
         logger.info("##### SOLR REINDEX START #####")
         logger.info(
-            "Reindexing {} items.".format(
-                brains_to_reindex.actual_result_count
-            )
+            "Reindexing {} items.".format(brains_to_reindex.actual_result_count)
         )
         skipped = 0
         indexed = 0
@@ -218,7 +211,10 @@ class ReindexBaseView(BrowserView):
             brains_to_sync = pc()
         good_uids = [x.UID for x in brains_to_sync]
         solr_results = search(
-            query={"site_name": get_site_title(), "b_size": 100000000},
+            query={
+                "site_name": all_site_titles(),
+                "b_size": 100000000,
+            },
             fl="UID",
         )
         uids_to_remove = [
@@ -256,7 +252,10 @@ class ReindexBaseView(BrowserView):
         logger.info("##### SYNC CONTENTS TO SOLR #####")
         pc = api.portal.get_tool(name="portal_catalog")
         solr_docs = search(
-            query={"site_name": get_site_title(), "b_size": 100000000},
+            query={
+                "site_name": all_site_titles(),
+                "b_size": 100000000,
+            },
             fl="UID modified",
         ).docs
         solr_items = {item["UID"]: item["modified"] for item in solr_docs}
@@ -276,25 +275,19 @@ class ReindexBaseView(BrowserView):
         logger.info("Completed in {}.".format(elapsed_time))
         if len(results["indexed"]):
             logger.info(
-                "Indexed {} items (new or modified):".format(
-                    len(results["indexed"])
-                )
+                "Indexed {} items (new or modified):".format(len(results["indexed"]))
             )
             for path in results["indexed"]:
                 logger.info("- {}".format(path))
         if len(results["removed"]):
             logger.info(
-                "Removed {} items (no more indexeable):".format(
-                    len(results["removed"])
-                )
+                "Removed {} items (no more indexeable):".format(len(results["removed"]))
             )
             for path in results["removed"]:
                 logger.info("- {}".format(path))
         if len(results["not_indexed"]):
             logger.info(
-                "NOT indexed {} items (errors):".format(
-                    len(results["not_indexed"])
-                )
+                "NOT indexed {} items (errors):".format(len(results["not_indexed"]))
             )
             for error in results["not_indexed"]:
                 logger.info(
@@ -304,9 +297,7 @@ class ReindexBaseView(BrowserView):
                     )
                 )
 
-    def sync_contents(
-        self, brains_to_sync, solr_items, disable_progress=False
-    ):
+    def sync_contents(self, brains_to_sync, solr_items, disable_progress=False):
         indexed = []
         not_indexed = []
         removed = []
@@ -335,11 +326,8 @@ class ReindexBaseView(BrowserView):
                         {"path": brain.getPath(), "message": e.__str__()}
                     )
                 except Exception as e:
-                    not_indexed.append(
-                        {"path": brain.getPath(), "message": str(e)}
-                    )
+                    not_indexed.append({"path": brain.getPath(), "message": str(e)})
             else:
-                item = brain.getObject()
                 if not can_index(item):
                     remove_from_solr(brain.UID)
                     removed.append(brain.getPath())
@@ -354,9 +342,7 @@ class ReindexBaseView(BrowserView):
                         if res:
                             indexed.append(brain.getPath())
                     except Exception as e:
-                        not_indexed.append(
-                            {"path": brain.getPath(), "message": str(e)}
-                        )
+                        not_indexed.append({"path": brain.getPath(), "message": str(e)})
         if not disable_progress:
             status["in_progress"] = False
         return {

{rer.solrpush-1.4.2 → rer.solrpush-1.5.0}/src/rer/solrpush/browser/querybuilder.py

@@ -22,7 +22,8 @@ logger = logging.getLogger(__name__)
 
 
 class QueryBuilder(BaseView):
-    def _makequery(
+
+    def _makequery(  # noqa: C901
         self,
         query=None,
         batch=False,
@@ -33,7 +34,7 @@ class QueryBuilder(BaseView):
         limit=0,
         brains=False,
         custom_query=None,
-    ):
+    ):  # noqa: C901
         """Parse the (form)query and return using multi-adapter"""
         query_modifiers = getUtilitiesFor(IQueryModifier)
         for name, modifier in sorted(query_modifiers, key=itemgetter(0)):
@@ -56,16 +57,12 @@ class QueryBuilder(BaseView):
 
         # Check for valid indexes
         catalog = getToolByName(self.context, "portal_catalog")
-        valid_indexes = [
-            index for index in parsedquery if index in catalog.indexes()
-        ]
+        valid_indexes = [index for index in parsedquery if index in catalog.indexes()]
 
         # We'll ignore any invalid index, but will return an empty set if none
         # of the indexes are valid.
         if not valid_indexes:
-            logger.warning(
-                "Using empty query because there are no valid indexes used."
-            )
+            logger.warning("Using empty query because there are no valid indexes used.")
             parsedquery = {}
 
         empty_query = not parsedquery  # store emptiness
@@ -105,6 +102,8 @@ class QueryBuilder(BaseView):
 
         if not empty_query:
             if search_with_solr:
+                parsedquery["b_start"] = b_start
+                parsedquery["b_size"] = b_size
                 if "SearchableText" in parsedquery:
                     if isinstance(parsedquery["SearchableText"], dict):
                         parsedquery["SearchableText"]["query"] = parsedquery[
@@ -115,9 +114,7 @@ class QueryBuilder(BaseView):
                             "SearchableText"
                         ].rstrip("*")
                 results = SolrResponse(
-                    data=solr_search(
-                        **self.clean_query_for_solr(query=parsedquery)
-                    )
+                    data=solr_search(**self.clean_query_for_solr(query=parsedquery))
                 )
             else:
                 results = catalog(**parsedquery)

{rer.solrpush-1.4.2 → rer.solrpush-1.5.0}/src/rer/solrpush/browser/scales.py

@@ -21,7 +21,7 @@ class SolrScalesHandler(BrowserView):
         title=None,
         direction="thumbnail",
         loading="lazy",
-        **args
+        **args,
     ):
         if not title:
             title = self.context.Title

{rer.solrpush-1.4.2 → rer.solrpush-1.5.0}/src/rer/solrpush/indexer.py

@@ -20,7 +20,7 @@ from zope.globalrequest import getRequest
 from zope.interface import implementer
 
 import logging
-
+import json
 
 logger = logging.getLogger(__name__)
 
@@ -45,13 +45,17 @@ class SolrIndexProcessor(object):
 
     @property
     def index_fields(self):
-        return api.portal.get_registry_record(
-            "index_fields", interface=IRerSolrpushSettings, default=None
+        indexes = api.portal.get_registry_record(
+            "index_fields", interface=IRerSolrpushSettings, default="{}"
         )
+        if not indexes:
+            return {}
+        return json.loads(indexes)
 
     def index(self, obj, attributes=None):
         if (
             self.active
+            and self.check_attributes(attributes=attributes)  # noqa
             and getattr(obj, "showinsearch", True)  # noqa
             and can_index(obj)  # noqa
         ):
@@ -63,8 +67,28 @@ class SolrIndexProcessor(object):
             return True
         return False
 
+    def check_attributes(self, attributes):
+        """
+        If we are reindexing some indexes that should not be indexed on SOLR,
+        we don't have to update it.
+        """
+        if not attributes:
+            return True
+        index_fields = api.portal.get_registry_record(
+            "index_fields", interface=IRerSolrpushSettings, default="{}"
+        )
+        if not index_fields:
+            return False
+        index_fields = json.loads(index_fields)
+
+        found = False
+        for index in attributes:
+            if index in index_fields:
+                found = True
+        return found
+
     def reindex(self, obj, attributes=None, update_metadata=None):
-        if self.active:
+        if self.active and self.check_attributes(attributes=attributes):
             if not self.index(obj, attributes):
                 self.unindex(obj)
 

{rer.solrpush-1.4.2 → rer.solrpush-1.5.0}/src/rer/solrpush/interfaces/elevate.py

@@ -1,11 +1,8 @@
 # -*- coding: utf-8 -*-
 from plone.app.vocabularies.catalog import CatalogSource as CatalogSourceBase
-from plone.app.z3cform.widget import RelatedItemsFieldWidget
-from plone.autoform import directives as form
 from plone.supermodel import model
 from rer.solrpush import _
-from z3c.relationfield.schema import RelationChoice
-from z3c.relationfield.schema import RelationList
+from zExceptions import BadRequest
 from zope import schema
 from zope.globalrequest import getRequest
 from zope.i18n import translate
@@ -24,32 +21,6 @@ class CatalogSource(CatalogSourceBase):
         return True  # Always contains to allow lazy handling of removed objs
 
 
-class IElevateRowSchema(model.Schema):
-    text = schema.List(
-        title=_("elevate_row_schema_text_label", default="Text"),
-        description=_(
-            "elevate_row_schema_text_help",
-            default="The word that should match in the search.",
-        ),
-        required=True,
-        value_type=schema.TextLine(),
-    )
-    uid = RelationList(
-        title=_("elevate_row_schema_uid_label", "Elements"),
-        description=_(
-            "elevate_row_schema_uid_help",
-            "Select a list of elements to elevate for that search word.",
-        ),
-        value_type=RelationChoice(vocabulary="plone.app.vocabularies.Catalog"),
-        required=True,
-    )
-    form.widget(
-        "uid",
-        RelatedItemsFieldWidget,
-        vocabulary="plone.app.vocabularies.Catalog",
-    )
-
-
 class IElevateSettings(model.Schema):
     """ """
 
@@ -64,12 +35,12 @@ class IElevateSettings(model.Schema):
 
     @invariant
     def elevate_invariant(data):
-        schema = json.loads(data.elevate_schema)
+        elevate_schema = json.loads(data.elevate_schema)
         request = getRequest()
-        words_mapping = [x["text"] for x in schema]
-        for i, schema_item in enumerate(schema):
-            elevate_text = schema_item.get("text", [])
-            if not elevate_text:
+        words_mapping = [x["keywords"] for x in elevate_schema]
+        for i, schema_item in enumerate(elevate_schema):
+            keywords = schema_item.get("keywords", [])
+            if not keywords:
                 raise Invalid(
                     translate(
                         _(
@@ -80,13 +51,13 @@ class IElevateSettings(model.Schema):
                         context=request,
                     )
                 )
-            for text in elevate_text:
+            for text in keywords:
                 for words_i, words in enumerate(words_mapping):
                     if i == words_i:
                         # it's the current config
                         continue
                     if text in words:
-                        raise Invalid(
+                        raise BadRequest(
                             translate(
                                 _(
                                     "text_duplicated_label",

{rer.solrpush-1.4.2 → rer.solrpush-1.5.0}/src/rer/solrpush/interfaces/settings.py

@@ -1,7 +1,5 @@
 # -*- coding: utf-8 -*-
 from plone.app.vocabularies.catalog import CatalogSource as CatalogSourceBase
-from plone.app.z3cform.widget import RelatedItemsFieldWidget
-from plone.autoform import directives as form
 from plone.supermodel import model
 from rer.solrpush import _
 from zope import schema
@@ -17,27 +15,6 @@ class CatalogSource(CatalogSourceBase):
         return True  # Always contains to allow lazy handling of removed objs
 
 
-class IElevateRowSchema(model.Schema):
-    text = schema.TextLine(
-        title=_("elevate_row_schema_text_label", default="Text"),
-        description=_(
-            "elevate_row_schema_text_help",
-            default="The word that should match in the search.",
-        ),
-        required=True,
-    )
-    uid = schema.List(
-        title=_("elevate_row_schema_uid_label", "Elements"),
-        description=_(
-            "elevate_row_schema_uid_help",
-            "Select a list of elements to elevate for that search word.",
-        ),
-        value_type=schema.Choice(source=CatalogSource()),
-        required=True,
-    )
-    form.widget("uid", RelatedItemsFieldWidget)
-
-
 class IRerSolrpushConf(model.Schema):
     """"""
 
@@ -49,9 +26,7 @@ class IRerSolrpushConf(model.Schema):
     )
     search_enabled = schema.Bool(
         title=_("Search enabled"),
-        description=_(
-            "Site search will use SOLR as engine instead portal_catalog."
-        ),
+        description=_("Site search will use SOLR as engine instead portal_catalog."),
         required=False,
         default=True,
     )
@@ -92,16 +67,13 @@ class IRerSolrpushConf(model.Schema):
         required=False,
         default=[],
         missing_value=[],
-        value_type=schema.Choice(
-            vocabulary="plone.app.vocabularies.PortalTypes"
-        ),
+        value_type=schema.Choice(vocabulary="plone.app.vocabularies.PortalTypes"),
     )
 
     index_fields = schema.SourceText(
         title=_(
             "index_fields_label",
-            default="List of fields loaded from SOLR that we use for "
-            "indexing.",
+            default="List of fields loaded from SOLR that we use for " "indexing.",
         ),
         description=_(
             "index_fields_help",

{rer.solrpush-1.4.2 → rer.solrpush-1.5.0}/src/rer/solrpush/parser.py

@@ -108,7 +108,7 @@ class Brain(dict):
         If site_name is the current site, convert the physical path into a url, if it was stored.
         Else return url attribute stored in SOLR
         """
-        url = self.get("url", "")
+        url = self.get("url", "") or self.get("@id", "")
         if self.is_current_site:
             frontend_url = api.portal.get_registry_record(
                 "frontend_url", interface=IRerSolrpushSettings, default=""

{rer.solrpush-1.4.2 → rer.solrpush-1.5.0}/src/rer/solrpush/profiles/default/metadata.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <metadata>
-  <version>2000</version>
+  <version>3000</version>
   <dependencies>
     <dependency>profile-collective.z3cform.jsonwidget:default</dependency>
     <dependency>profile-plone.restapi:default</dependency>

{rer.solrpush-1.4.2 → rer.solrpush-1.5.0}/src/rer/solrpush/restapi/configure.zcml

@@ -8,6 +8,7 @@
       file="configure.zcml"
       />
   <include package=".deserializer" />
+  <include package=".serializer" />
   <include package=".services" />
 
 

{rer.solrpush-1.4.2 → rer.solrpush-1.5.0}/src/rer/solrpush/restapi/deserializer/controlpanel.py

@@ -1,6 +1,4 @@
-from plone.restapi.deserializer.controlpanels import (
-    ControlpanelDeserializeFromJson,
-)
+from plone.restapi.deserializer.controlpanels import ControlpanelDeserializeFromJson
 from plone.restapi.interfaces import IDeserializeFromJson
 from rer.solrpush.interfaces import IRERSolrpushRestapiControlpanel
 from rer.solrpush.utils import init_solr_push
@@ -10,9 +8,7 @@ from zope.interface import implementer
 
 @implementer(IDeserializeFromJson)
 @adapter(IRERSolrpushRestapiControlpanel)
-class RERSolrpushControlpanelDeserializeFromJson(
-    ControlpanelDeserializeFromJson
-):
+class RERSolrpushControlpanelDeserializeFromJson(ControlpanelDeserializeFromJson):
     def __call__(self):
         super().__call__()
         init_solr_push()