rer.solrpush 1.3.4__py3-none-any.whl

rer/solrpush/tests/test_vocabularies.py

@@ -0,0 +1,113 @@
+# -*- coding: utf-8 -*-
+"""Setup tests for this package."""
+from plone.app.testing import setRoles
+from plone.app.testing import TEST_USER_ID
+from rer.solrpush.utils.solr_common import init_solr_push
+from rer.solrpush.utils.solr_common import get_solr_connection
+from rer.solrpush.utils import push_to_solr
+from rer.solrpush.testing import RER_SOLRPUSH_API_FUNCTIONAL_TESTING
+from transaction import commit
+from zope.component import getUtility
+from zope.schema.interfaces import IVocabularyFactory
+
+import unittest
+
+
+class TestVocabularies(unittest.TestCase):
+    """Test show in search field behavior."""
+
+    layer = RER_SOLRPUSH_API_FUNCTIONAL_TESTING
+
+    def setUp(self):
+        """
+        We create objects before initializing solr settings, so solr core is
+        always empty on setUp.
+        """
+        self.portal = self.layer["portal"]
+        self.request = self.layer["request"]
+        setRoles(self.portal, TEST_USER_ID, ["Manager"])
+
+        init_solr_push()
+        commit()
+
+        push_to_solr(
+            {
+                "title": "Document from site foo",
+                "portal_type": "Document",
+                "site_name": "foo",
+                "UID": "000",
+            }
+        )
+        push_to_solr(
+            {
+                "title": "Document from site foo with subject",
+                "portal_type": "Document",
+                "site_name": "foo",
+                "Subject": ["foo-subject"],
+                "UID": "111",
+            }
+        )
+        push_to_solr(
+            {
+                "title": "Document from site bar",
+                "portal_type": "Document",
+                "site_name": "bar",
+                "UID": "222",
+            }
+        )
+        push_to_solr(
+            {
+                "title": "News Item from site bar",
+                "portal_type": "News Item",
+                "site_name": "bar",
+                "Subject": ["bar-subject"],
+                "UID": "333",
+            }
+        )
+
+    def tearDown(self):
+        solr = get_solr_connection()
+        solr.delete(q="*:*", commit=True)
+        commit()
+
+    def test_sites_vocabulary_return_all_stored_ones(self):
+        factory = getUtility(
+            IVocabularyFactory, "rer.solrpush.vocabularies.AvailableSites"
+        )
+        vocabulary = factory(self.portal)
+
+        self.assertEqual(len(vocabulary), 2)
+        self.assertEqual([x.title for x in vocabulary], ["bar", "foo"])
+        self.assertEqual(vocabulary.getTerm("foo").title, "foo")
+        self.assertEqual(vocabulary.getTerm("bar").title, "bar")
+
+    def test_Subject_vocabulary_return_all_stored_ones(self):
+        factory = getUtility(
+            IVocabularyFactory, "rer.solrpush.vocabularies.AvailableSubjects"
+        )
+        vocabulary = factory(self.portal)
+
+        self.assertEqual(len(vocabulary), 2)
+        self.assertEqual(
+            [x.title for x in vocabulary], ["bar-subject", "foo-subject"]
+        )
+        self.assertEqual(
+            vocabulary.getTerm("foo-subject").title, "foo-subject"
+        )
+        self.assertEqual(
+            vocabulary.getTerm("bar-subject").title, "bar-subject"
+        )
+
+    def test_portal_type_vocabulary_return_all_stored_ones(self):
+        factory = getUtility(
+            IVocabularyFactory,
+            "rer.solrpush.vocabularies.AvailablePortalTypes",
+        )
+        vocabulary = factory(self.portal)
+
+        self.assertEqual(len(vocabulary), 2)
+        self.assertEqual(
+            [x.title for x in vocabulary], ["Document", "News Item"]
+        )
+        self.assertEqual(vocabulary.getTerm("Document").title, "Document")
+        self.assertEqual(vocabulary.getTerm("News Item").title, "News Item")

rer/solrpush/upgrades.py

@@ -0,0 +1,76 @@
+# -*- coding: utf-8 -*-
+from plone import api
+from plone.app.upgrade.utils import installOrReinstallProduct
+from rer.solrpush.utils.solr_common import init_solr_push
+from rer.solrpush.utils.solr_indexer import push_to_solr
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+default_profile = "profile-rer.solrpush:default"
+
+
+def update_profile(context, name):
+    setup_tool = api.portal.get_tool(name="portal_setup")
+    setup_tool.runImportStepFromProfile(default_profile, name)
+
+
+def update_registry(context):
+    update_profile(context=context, name="plone.app.registry")
+
+
+def update_controlpanel(context):
+    update_profile(context=context, name="controlpanel")
+
+
+def update_actions(context):
+    update_profile(context=context, name="actions")
+
+
+def to_1100(context):
+    setup_tool = api.portal.get_tool(name="portal_setup")
+    setup_tool.runAllImportStepsFromProfile("profile-rer.solrpush:to_1100")
+    setup_tool.runImportStepFromProfile(default_profile, "plone.app.registry")
+
+
+def to_1200(context):
+    setup_tool = api.portal.get_tool(name="portal_setup")
+    setup_tool.runImportStepFromProfile(default_profile, "plone.app.registry")
+    setup_tool.runImportStepFromProfile(default_profile, "rolemap")
+    setup_tool.runImportStepFromProfile(default_profile, "controlpanel")
+
+
+def to_1300(context):
+    setup_tool = api.portal.get_tool(name="portal_setup")
+    setup_tool.runAllImportStepsFromProfile("profile-rer.solrpush:to_1300")
+
+
+def to_1400(context):
+    installOrReinstallProduct(
+        api.portal.get(), "collective.z3cform.jsonwidget"
+    )
+    update_registry(context)
+    update_controlpanel(context)
+
+
+def to_1600(context):
+    # add new criteria
+    update_registry(context)
+    # reload schema
+    init_solr_push()
+    brains = api.content.find(portal_type=["File", "Image"])
+    tot = len(brains)
+    logger.info("Reindexing {} contents for updated mime_types.".format(tot))
+    i = 0
+
+    for brain in brains:
+        i += 1
+        if i % 500 == 0:
+            logger.info("[PROGRESS] - {}/{}".format(i, tot))
+        item = brain.getObject()
+        try:
+            push_to_solr(item)
+        except Exception:
+            # solr can't index it, pass
+            continue

rer/solrpush/upgrades.zcml

@@ -0,0 +1,86 @@
+<configure
+  xmlns="http://namespaces.zope.org/zope"
+  xmlns:genericsetup="http://namespaces.zope.org/genericsetup">
+
+  <genericsetup:upgradeStep
+    source="1000"
+    destination="1100"
+    title="Remove persistent utility"
+    description=""
+    profile="rer.solrpush:default"
+    handler=".upgrades.to_1100"
+    />
+
+  <genericsetup:upgradeStep
+    source="1100"
+    destination="1200"
+    title="Added css resource and bundle"
+    description="Styles for controlpanel"
+    profile="rer.solrpush:default"
+    handler=".upgrades.to_1200"
+    />
+  <genericsetup:upgradeStep
+    source="1200"
+    destination="1300"
+    title="Remove unused control panel"
+    description=""
+    profile="rer.solrpush:default"
+    handler=".upgrades.to_1300"
+    />
+  <genericsetup:upgradeStep
+    source="1300"
+    destination="1400"
+    title="Remove unused control panel"
+    description=""
+    profile="rer.solrpush:default"
+    handler=".upgrades.to_1400"
+    />
+  <genericsetup:upgradeStep
+    source="1400"
+    destination="1500"
+    title="Add new field in controlpanel: force_commit"
+    description=""
+    profile="rer.solrpush:default"
+    handler=".upgrades.update_registry"
+    />
+  <genericsetup:upgradeStep
+    source="1500"
+    destination="1600"
+    title="Add new criteria and reindex files"
+    description=""
+    profile="rer.solrpush:default"
+    handler=".upgrades.to_1600"
+    />
+  <genericsetup:upgradeStep
+    source="1600"
+    destination="1700"
+    title="Add new field in controlpanel: query_debug"
+    description=""
+    profile="rer.solrpush:default"
+    handler=".upgrades.update_registry"
+    />  
+  <genericsetup:upgradeStep
+    source="1700"
+    destination="1800"
+    title="Add new criteria: solr_portal_types"
+    description=""
+    profile="rer.solrpush:default"
+    handler=".upgrades.update_registry"
+    />
+  <genericsetup:upgradeStep
+    source="1800"
+    destination="1900"
+    title="Add elevate control panel link in user actions"
+    description=""
+    profile="rer.solrpush:default"
+    handler=".upgrades.update_actions"
+    />
+  <genericsetup:upgradeStep
+    source="1900"
+    destination="2000"
+    title="Add search_enabled setting field"
+    description=""
+    profile="rer.solrpush:default"
+    handler=".upgrades.update_registry"
+    />  
+</configure>

rer/solrpush/utils/__init__.py

@@ -0,0 +1,6 @@
+# -*- coding: utf-8 -*-
+from rer.solrpush.utils.solr_common import init_solr_push  # noqa
+from rer.solrpush.utils.solr_indexer import push_to_solr  # noqa
+from rer.solrpush.utils.solr_indexer import remove_from_solr  # noqa
+from rer.solrpush.utils.solr_indexer import reset_solr  # noqa
+from rer.solrpush.utils.solr_search import search  # noqa

rer/solrpush/utils/solr_common.py

@@ -0,0 +1,122 @@
+# -*- coding: utf-8 -*-
+from lxml import etree
+from plone import api
+from rer.solrpush import _
+from rer.solrpush.interfaces.settings import IRerSolrpushSettings
+
+import json
+import pysolr
+import requests
+import six
+
+
+def get_setting(field, interface=IRerSolrpushSettings):
+    return api.portal.get_registry_record(
+        field, interface=interface, default=False
+    )
+
+
+def set_setting(field, value, interface=IRerSolrpushSettings):
+    return api.portal.set_registry_record(
+        field, interface=interface, value=value
+    )
+
+
+def is_solr_active():
+    """Just checking if solr indexing is set to active in control panel"""
+    return get_setting(field="active")
+
+
+def should_force_commit():
+    return get_setting(field="force_commit") or False
+
+
+def get_index_fields():
+    json_str = get_setting(field="index_fields")
+    return json.loads(json_str)
+
+
+def extract_fields(nodes):
+    fields = {}
+    for node in nodes:
+        field_name = node.get("name")
+        field_type = node.get("type")
+        if six.PY2:
+            field_name = six.text_type(field_name)
+            field_type = six.text_type(field_type)
+        fields[field_name] = {"type": field_type}
+    return fields
+
+
+def init_solr_push():
+    """Inizializza la voce di registro 'index_fields'
+
+    Lo fa leggendo il file xml di SOLR.
+
+    :param solr_url: [required] L'url a cui richiedere il file xml
+    :type solr_url: string
+    :returns: Empty String if everything's good
+    :rtype: String
+    """
+    solr_url = get_setting(field="solr_url")
+
+    if solr_url:
+        if not solr_url.endswith("/"):
+            solr_url = solr_url + "/"
+        try:
+            respo = requests.get(solr_url + "admin/file?file=schema.xml")
+        except requests.exceptions.RequestException as err:
+            ErrorMessage = "Connection problem:\n{0}".format(err)
+            return ErrorMessage
+        if respo.status_code != 200:
+            ErrorMessage = "Problems fetching schema:\n{0}\n{1}".format(
+                respo.status_code, respo.reason
+            )
+            return ErrorMessage
+
+        root = etree.fromstring(respo.content)
+        chosen_fields = json.dumps(
+            extract_fields(nodes=root.findall(".//field"))
+        )
+        if six.PY2:
+            chosen_fields = chosen_fields.decode("utf-8")
+        set_setting(field="index_fields", value=chosen_fields)
+        set_setting(field="ready", value=True)
+        return
+
+    return _("No SOLR url provided")
+
+
+def get_solr_connection(context=None, **kwargs):
+    # TODO: fix sporadic
+    # ResourceWarning: Enable tracemalloc to get the object allocation traceback
+    # ResourceWarning: unclosed <socket.socket ... raddr=('127.0.0.1', 8983)>
+    # -> avoid explicit close connection
+
+    # XXX: rivalutare il default True per always_commit !
+    if "always_commit" not in kwargs:
+        kwargs["always_commit"] = True
+    is_ready = get_setting(field="ready")
+    solr_url = get_setting(field="solr_url")
+    if not is_ready or not solr_url:
+        return
+    if context is None:
+        context = api.portal.get()
+    if context is None:
+        client = pysolr.Solr(solr_url, **kwargs)
+    else:
+        jar = getattr(context, "_p_jar", None)
+        oid = getattr(context, "_p_oid", None)
+        if jar is None or oid is None:
+            # object is not persistent or is not yet associated with a
+            # connection
+            cache = context._v_solr_client_cache = {}
+        else:
+            cache = getattr(jar, "foreign_connections", None)
+            if cache is None:
+                cache = jar.foreign_connections = {}
+        cache_key = "solr_%s_%s" % (solr_url, kwargs)
+        client = cache.get(cache_key)
+        if client is None:
+            client = cache[cache_key] = pysolr.Solr(solr_url, **kwargs)
+    return client

rer/solrpush/utils/solr_indexer.py

@@ -0,0 +1,295 @@
+# -*- coding: utf-8 -*-
+from DateTime import DateTime
+from plone import api
+from plone.indexer.interfaces import IIndexableObject
+from plone.registry.interfaces import IRegistry
+from rer.solrpush import _
+from rer.solrpush.interfaces.adapter import IExtractFileFromTika
+from six.moves import map
+from zope.component import getUtility
+from zope.component import queryMultiAdapter
+from rer.solrpush.utils.solr_common import get_solr_connection
+from rer.solrpush.utils.solr_common import get_setting
+from rer.solrpush.utils.solr_common import get_index_fields
+from rer.solrpush.utils.solr_common import is_solr_active
+from rer.solrpush.utils.solr_common import should_force_commit
+
+import datetime
+import logging
+import pysolr
+import six
+
+
+if six.PY2:
+    from ftfy import fix_text
+
+try:
+    # rer.agidtheme overrides site tile field
+    from rer.agidtheme.base.interfaces import IRERSiteSchema as ISiteSchema
+    from rer.agidtheme.base.utility.interfaces import ICustomFields
+
+    RER_THEME = True
+except ImportError:
+    from Products.CMFPlone.interfaces.controlpanel import ISiteSchema
+
+    RER_THEME = False
+
+logger = logging.getLogger(__name__)
+
+ADDITIONAL_FIELDS = ["searchwords"]
+
+RESTAPI_METADATA_FIELDS = ["@id", "@type", "description", "title"]
+
+# HELPER METHODS
+
+
+def get_site_title():
+    registry = getUtility(IRegistry)
+    site_settings = registry.forInterface(
+        ISiteSchema, prefix="plone", check=False
+    )
+    site_title = getattr(site_settings, "site_title") or ""
+    if RER_THEME:
+        site_subtitle_style = (
+            getattr(site_settings, "site_subtitle_style") or ""
+        )
+        fields_value = getUtility(ICustomFields)
+        site_title = fields_value.titleLang(site_title)
+        site_subtitle = fields_value.subtitleLang(
+            getattr(site_settings, "site_subtitle") or "{}"
+        )
+        if site_subtitle and site_subtitle_style == "subtitle-normal":
+            site_title += " {}".format(site_subtitle)
+
+    if six.PY2:
+        site_title = site_title.decode("utf-8")
+    return site_title
+
+
+def parse_date_as_datetime(value):
+    """Sistemiamo le date"""
+    if value:
+        format = "%Y-%m-%dT%H:%M:%S"
+        return value.utcdatetime().strftime(format) + "Z"
+    return value
+
+
+def parse_date_str(value):
+    if isinstance(value, datetime.date):
+        value = datetime.datetime.combine(value, datetime.time.min)
+    return parse_date_as_datetime(DateTime(value))
+
+
+def attachment_to_index(item):
+    """
+    If item has a provider to extract text, return the file to be indexed
+    """
+    try:
+        provider = IExtractFileFromTika(item)
+        return provider.get_file_to_index()
+    except TypeError:
+        return None
+
+
+def is_right_portal_type(item):
+    enabled_types = get_setting(field="enabled_types")
+    if not enabled_types:
+        return True
+    return item.portal_type in enabled_types
+
+
+def can_index(item):
+    """Check if the item passed as argument can and has to be indexed"""
+    with api.env.adopt_roles(["Anonymous"]):
+        if not api.user.has_permission("View", obj=item):
+            return False
+    if not is_solr_active():
+        return False
+    return is_right_portal_type(item)
+
+
+def create_index_dict(item):
+    """Restituisce un dizionario pronto per essere 'mandato' a SOLR per
+    l'indicizzazione.
+    """
+    index_fields = get_index_fields()
+    frontend_url = get_setting(field="frontend_url")
+    if frontend_url:
+        frontend_url = frontend_url.rstrip("/")
+
+    catalog = api.portal.get_tool(name="portal_catalog")
+    adapter = queryMultiAdapter((item, catalog), IIndexableObject)
+    index_me = {}
+    for field, field_infos in index_fields.items():
+        if field in RESTAPI_METADATA_FIELDS:
+            # skip. These fields are only metadata fields needed in restap-like
+            # repsonses and can be copied in solr configuration.
+            continue
+        field_type = field_infos.get("type")
+        if six.PY2:
+            field = field.encode("ascii")
+        value = getattr(adapter, field, None)
+        if not value and value is not False:
+            continue
+        if callable(value):
+            value = value()
+        if six.PY2:
+            value = fix_py2_strings(value)
+        if isinstance(value, DateTime):
+            value = parse_date_as_datetime(value)
+        else:
+            if field_type == "date":
+                value = parse_date_str(value)
+        index_me[field] = value
+
+    for field in ADDITIONAL_FIELDS:
+        value = getattr(item, field, None)
+        if value is not None:
+            index_me[field] = value
+    portal = api.portal.get()
+
+    # extra-catalog schema
+    index_me["site_name"] = get_site_title()
+    index_me["path"] = "/".join(item.getPhysicalPath())
+    index_me["path_depth"] = len(item.getPhysicalPath()) - 2
+    index_me["attachment"] = attachment_to_index(item)
+    index_me["getIcon"] = False
+    if "view_name" in index_fields.keys():
+        index_me["view_name"] = item.getLayout()
+
+    # convert url to frontend one
+    if frontend_url:
+        index_me["url"] = item.absolute_url().replace(
+            portal.portal_url(), frontend_url
+        )
+    else:
+        index_me["url"] = item.absolute_url()
+
+    has_image = getattr(item.aq_base, "image", None)
+    if has_image:
+        index_me["getIcon"] = True
+
+    return index_me
+
+
+def fix_py2_strings(value):
+    """REMOVE ON PYTHON 3"""
+    if isinstance(value, six.string_types):
+        if not isinstance(value, six.text_type):
+            value = value.replace("\xc0?", "").decode("utf-8", "ignore")
+        return fix_text(value)
+    if isinstance(value, list):
+        return list(map(fix_py2_strings, value))
+    if isinstance(value, tuple):
+        return tuple(map(fix_py2_strings, value))
+    return value
+
+
+def encode_strings_for_attachments(value):
+    """
+    Needed because attachments POST use standard requests encoding
+    and not utf-8
+    """
+    if isinstance(value, six.string_types):
+        return value.encode("ISO-8859-1", "ignore")
+    if isinstance(value, list):
+        return list(map(encode_strings_for_attachments, value))
+    if isinstance(value, tuple):
+        return tuple(map(encode_strings_for_attachments, value))
+    return value
+
+
+def add_with_attachment(solr, attachment, fields):
+    params = {
+        "extractOnly": "false",
+        "lowernames": "false",
+        "fmap.content": "SearchableText",
+        "fmap.title": "SearchableText",
+        "literalsOverride": "true",
+        "commit": should_force_commit() and "true" or "false",
+    }
+    params.update(
+        {
+            "literal.{key}".format(key=key): encode_strings_for_attachments(
+                value
+            )
+            for (key, value) in fields.items()
+        }
+    )
+    path = "update/extract"
+    return solr._send_request(
+        method="post",
+        path=path,
+        body=params,
+        files={"file": ("extracted", attachment)},
+    )
+
+
+# END HELPER METHODS
+
+# LIBRARY METHODS
+
+
+def push_to_solr(item_or_obj):
+    """
+    Perform push to solr
+    """
+    solr = get_solr_connection()
+    if not solr:
+        logger.error("Unable to push to solr. Configuration is incomplete.")
+        return
+    context = item_or_obj
+    if isinstance(item_or_obj, dict):
+        obj = api.content.get(UID=item_or_obj.get("UID", ""))
+        if obj:
+            context = obj
+    if not can_index(context):
+        return False
+    index_dict = create_index_dict(context)
+    attachment = index_dict.pop("attachment", None)
+    if attachment:
+        add_with_attachment(
+            solr=solr, attachment=attachment, fields=index_dict
+        )
+    else:
+        solr.add(docs=[index_dict], commit=should_force_commit())
+    return True
+
+
+def remove_from_solr(uid):
+    """
+    Perform remove item from solr
+    """
+    if not is_solr_active():
+        return
+    solr = get_solr_connection()
+    portal = api.portal.get()
+    if not solr:
+        logger.error("Unable to push to solr. Configuration is incomplete.")
+        return
+    try:
+        solr.delete(
+            q="UID:{}".format(uid),
+            commit=should_force_commit(),
+        )
+    except (pysolr.SolrError, TypeError) as err:
+        logger.error(err)
+        message = _(
+            "content_remove_error",
+            default=u"There was a problem removing this content from SOLR. "
+            " Please contact site administrator.",
+        )
+        api.portal.show_message(
+            message=message, request=portal.REQUEST, type="warning"
+        )
+
+
+def reset_solr():
+    solr = get_solr_connection()
+    if not solr:
+        logger.error("Unable to push to solr. Configuration is incomplete.")
+        return
+    solr.delete(
+        q='site_name:"{}"'.format(get_site_title()),
+        commit=should_force_commit(),
+    )