rer.solrpush 1.3.4__py3-none-any.whl

rer/solrpush/testing.py

@@ -0,0 +1,161 @@
+# -*- coding: utf-8 -*-
+from plone.api.portal import set_registry_record
+from plone.app.contenttypes.testing import PLONE_APP_CONTENTTYPES_FIXTURE
+from plone.app.testing import applyProfile
+from plone.app.testing import FunctionalTesting
+from plone.app.testing import IntegrationTesting
+from plone.app.testing import PloneSandboxLayer
+from plone.restapi.testing import PloneRestApiDXLayer
+from plone.testing import Layer
+from plone.testing import z2
+from rer.solrpush.interfaces.settings import IRerSolrpushSettings
+from rer.solrpush.utils import init_solr_push
+from six.moves import range
+from time import sleep
+from ZPublisher.HTTPRequest import HTTPRequest
+
+import collective.z3cform.jsonwidget
+import os
+import plone.restapi
+import rer.solrpush
+import six
+import subprocess
+import sys
+
+BIN_DIR = os.path.dirname(os.path.realpath(sys.argv[0]))
+
+
+class SolrLayer(Layer):
+    """Solr test layer that fires up and shuts down a Solr instance. This
+    layer can be used to unit test a Solr configuration without having to
+    fire up Plone.
+    """
+
+    proc = None
+
+    def __init__(
+        self,
+        bases=None,
+        name="Solr Layer",
+        module=None,
+        solr_host="localhost",
+        solr_port=8983,
+        solr_base="/solr/solrpush_test",
+    ):
+        super(SolrLayer, self).__init__(bases, name, module)
+        self.solr_host = solr_host
+        self.solr_port = solr_port
+        self.solr_base = solr_base
+        self.solr_url = u"http://{0}:{1}{2}".format(
+            solr_host, solr_port, solr_base
+        )
+
+    def setUp(self):
+        """Start Solr and poll until it is up and running."""
+        self.proc = subprocess.call(
+            "./solr-start", shell=True, close_fds=True, cwd=BIN_DIR
+        )
+        # Poll Solr until it is up and running
+        solr_ping_url = "{0}/admin/ping?wt=xml".format(self.solr_url)
+        for i in range(1, 10):
+            try:
+                result = six.moves.urllib.request.urlopen(solr_ping_url)
+                if result.code == 200:
+                    if b'<str name="status">OK</str>' in result.read():
+                        break
+            except six.moves.urllib.error.URLError:
+                sleep(3)
+                sys.stdout.write(".")
+            if i == 9:
+                subprocess.call(
+                    "./solr-stop", shell=True, close_fds=True, cwd=BIN_DIR
+                )
+                sys.stdout.write("Solr Instance could not be started !!!")
+
+    def tearDown(self):
+        """Stop Solr."""
+        solr_clean_url = "{0}/update?stream.body=<delete><query>*:*</query></delete>&commit=true".format(  # noqa
+            self.solr_url
+        )
+        six.moves.urllib.request.urlopen(solr_clean_url)
+        subprocess.check_call(
+            "./solr-stop", shell=True, close_fds=True, cwd=BIN_DIR
+        )
+
+
+SOLR_FIXTURE = SolrLayer()
+
+
+class RerSolrpushRestApiLayer(PloneRestApiDXLayer):
+    """ """
+
+    defaultBases = (PLONE_APP_CONTENTTYPES_FIXTURE,)
+
+    def __init__(
+        self,
+        bases=None,
+        name="SolrPush Layer",
+        module=None,
+        solr_host=u"localhost",
+        solr_port=8983,
+        solr_base=u"/solr/solrpush_test",
+    ):
+        super(PloneSandboxLayer, self).__init__(bases, name, module)
+        self.solr_host = solr_host
+        self.solr_port = solr_port
+        self.solr_base = solr_base
+        # SolrLayer should use the same settings as CollectiveSolrLayer
+        self.solr_layer = SolrLayer(
+            bases,
+            name,
+            module,
+            solr_host=solr_host,
+            solr_port=solr_port,
+            solr_base=solr_base,
+        )
+
+    def setUpZope(self, app, configurationContext):
+        # Load any other ZCML that is required for your tests.
+        # The z3c.autoinclude feature is disabled in the Plone fixture base
+        # layer.
+        self._orig_retry_max_count = HTTPRequest.retry_max_count
+        HTTPRequest.retry_max_count = 3
+
+        self.loadZCML(package=rer.solrpush)
+        self.loadZCML(package=plone.restapi)
+        self.loadZCML(package=collective.z3cform.jsonwidget)
+
+    def tearDownZope(self, app):
+        HTTPRequest.retry_max_count = self._orig_retry_max_count
+
+    def setUpPloneSite(self, portal):
+        self.solr_layer.setUp()
+        applyProfile(portal, "rer.solrpush:default")
+        set_registry_record("active", True, interface=IRerSolrpushSettings)
+        set_registry_record(
+            "solr_url",
+            self.solr_layer.solr_url,
+            interface=IRerSolrpushSettings,
+        )
+        init_solr_push()
+
+    def tearDownPloneSite(self, portal):
+        set_registry_record("active", True, interface=IRerSolrpushSettings)
+        set_registry_record(
+            "solr_url",
+            self.solr_layer.solr_url,
+            interface=IRerSolrpushSettings,
+        )
+        self.solr_layer.tearDown()
+
+
+RER_SOLRPUSH_API_FIXTURE = RerSolrpushRestApiLayer()
+RER_SOLRPUSH_API_INTEGRATION_TESTING = IntegrationTesting(
+    bases=(RER_SOLRPUSH_API_FIXTURE,),
+    name="RerSolrpushRestApiLayer:Integration",
+)
+
+RER_SOLRPUSH_API_FUNCTIONAL_TESTING = FunctionalTesting(
+    bases=(RER_SOLRPUSH_API_FIXTURE, z2.ZSERVER_FIXTURE),
+    name="RerSolrpushRestApiLayer:Functional",
+)

rer/solrpush/tests/test_behaviors.py

@@ -0,0 +1,61 @@
+# -*- coding: utf-8 -*-
+"""Setup tests for this package."""
+from plone import api
+from plone.api.portal import set_registry_record
+from plone.app.testing import setRoles
+from plone.app.testing import TEST_USER_ID
+from rer.solrpush.interfaces.settings import IRerSolrpushSettings
+from rer.solrpush.utils import init_solr_push
+from rer.solrpush.utils import reset_solr
+from rer.solrpush.utils import search
+from rer.solrpush.testing import (
+    RER_SOLRPUSH_API_FUNCTIONAL_TESTING,
+)  # noqa: E501
+from transaction import commit
+
+import unittest
+
+
+class TestShowInSearch(unittest.TestCase):
+    """Test show in search field behavior."""
+
+    layer = RER_SOLRPUSH_API_FUNCTIONAL_TESTING
+
+    def setUp(self):
+        """
+        We create objects before initializing solr settings, so solr core is
+        always empty on setUp.
+        """
+        self.portal = self.layer["portal"]
+        self.request = self.layer["request"]
+        setRoles(self.portal, TEST_USER_ID, ["Manager"])
+        set_registry_record(
+            "enabled_types",
+            [u"Document", u"News Item"],
+            interface=IRerSolrpushSettings,
+        )
+        init_solr_push()
+        self.document = api.content.create(
+            container=self.portal, type="Document", title="Document foo"
+        )
+        api.content.transition(obj=self.document, transition="publish")
+        self.news = api.content.create(
+            container=self.portal, type="News Item", title="News bar"
+        )
+        api.content.transition(obj=self.news, transition="publish")
+        commit()
+
+    def tearDown(self):
+        reset_solr()
+        commit()
+
+    def test_items_are_indexed_by_default(self):
+        solr_results = search(query={"*": "*", "b_size": 100000}, fl="UID")
+        self.assertEqual(solr_results.hits, 2)
+
+    def test_items_are_unindexed_when_set_false(self):
+        self.document.showinsearch = False
+        self.document.reindexObject()
+        commit()
+        solr_results = search(query={"*": "*", "b_size": 100000}, fl="UID")
+        self.assertEqual(solr_results.hits, 1)

rer/solrpush/tests/test_elevate.py

@@ -0,0 +1,124 @@
+# -*- coding: utf-8 -*-
+"""Setup tests for this package."""
+from plone import api
+from plone.api.portal import set_registry_record
+from plone.app.testing import setRoles
+from plone.app.testing import TEST_USER_ID
+from rer.solrpush.interfaces import IElevateSettings
+from rer.solrpush.interfaces.settings import IRerSolrpushSettings
+from rer.solrpush.utils import init_solr_push
+from rer.solrpush.utils import reset_solr
+from rer.solrpush.utils import search
+from rer.solrpush.testing import RER_SOLRPUSH_API_FUNCTIONAL_TESTING
+from transaction import commit
+
+import unittest
+import json
+import six
+
+
+class TestSolrSearch(unittest.TestCase):
+    """
+    """
+
+    layer = RER_SOLRPUSH_API_FUNCTIONAL_TESTING
+
+    def setUp(self):
+        """
+        """
+        self.portal = self.layer["portal"]
+        self.request = self.layer["request"]
+        setRoles(self.portal, TEST_USER_ID, ["Manager"])
+        set_registry_record(
+            "enabled_types",
+            [u"Document", u"News Item"],
+            interface=IRerSolrpushSettings,
+        )
+
+        init_solr_push()
+        commit()
+        self.doc1 = api.content.create(
+            container=self.portal,
+            type="Document",
+            title="First page",
+            description="lorem ipsum",
+            subject=["foo", "bar"],
+        )
+        self.doc2 = api.content.create(
+            container=self.portal,
+            type="Document",
+            title="Second page",
+            description="lorem ipsum dolor sit amet",
+            searchwords="important",
+        )
+        self.unpublished_doc = api.content.create(
+            container=self.portal, type="Document", title="Unpublished page"
+        )
+        self.published_news = api.content.create(
+            container=self.portal,
+            type="News Item",
+            title="Published News",
+            subject=["foo", "news category"],
+        )
+        self.unpublished_news = api.content.create(
+            container=self.portal,
+            type="News Item",
+            title="Unpublished News",
+            subject=["foo"],
+        )
+        self.event = api.content.create(
+            container=self.portal, type="Event", title="Event", subject=["foo"]
+        )
+        api.content.transition(obj=self.doc1, transition="publish")
+        api.content.transition(obj=self.doc2, transition="publish")
+        api.content.transition(obj=self.published_news, transition="publish")
+        api.content.transition(obj=self.event, transition="publish")
+        commit()
+
+    def tearDown(self):
+        set_registry_record("active", True, interface=IRerSolrpushSettings)
+        #  reset elevate
+        set_registry_record("elevate_schema", u"", interface=IElevateSettings)
+        reset_solr()
+        commit()
+
+    def test_search_with_elevate(self):
+        doc3 = api.content.create(
+            container=self.portal, type="Document", title="Third page"
+        )
+        # page with the shortest title containing keyword has the better score
+        doc4 = api.content.create(
+            container=self.portal, type="Document", title="page"
+        )
+        api.content.transition(obj=doc3, transition="publish")
+        api.content.transition(obj=doc4, transition="publish")
+        commit()
+
+        solr_results = search(
+            query={"SearchableText": "page"}, fl=["UID", "Title", "[elevated]"]
+        ).docs
+        self.assertEqual(len(solr_results), 4)
+        self.assertEqual(
+            solr_results[0],
+            {"UID": doc4.UID(), "Title": doc4.Title(), "[elevated]": False},
+        )
+
+        # now let's set an elevate for third document
+        value = json.dumps([{"text": [u"page"], "uid": [{"UID": doc3.UID()}]}])
+        if six.PY2:
+            value = value.decode("utf-8")
+        set_registry_record(
+            "elevate_schema", value, interface=IElevateSettings,
+        )
+        solr_results = search(
+            query={"SearchableText": "page"}, fl=["UID", "Title", "[elevated]"]
+        ).docs
+        self.assertEqual(len(solr_results), 4)
+        self.assertEqual(
+            solr_results[0],
+            {"UID": doc3.UID(), "Title": doc3.Title(), "[elevated]": True},
+        )
+        self.assertEqual(
+            solr_results[1],
+            {"UID": doc4.UID(), "Title": doc4.Title(), "[elevated]": False},
+        )

rer/solrpush/tests/test_events.py

@@ -0,0 +1,181 @@
+# -*- coding: utf-8 -*-
+"""Setup tests for this package."""
+from plone import api
+from plone.api.portal import get_registry_record
+from plone.api.portal import set_registry_record
+from plone.app.testing import setRoles
+from plone.app.testing import TEST_USER_ID
+from rer.solrpush.interfaces.settings import IRerSolrpushSettings
+from rer.solrpush.utils import init_solr_push
+from rer.solrpush.testing import (
+    RER_SOLRPUSH_API_FUNCTIONAL_TESTING,
+)  # noqa: E501
+from transaction import commit
+from zope.event import notify
+from zope.lifecycleevent import ObjectModifiedEvent
+from rer.solrpush.utils import reset_solr
+
+import requests
+import unittest
+
+try:
+    from Products.CMFPlone.utils import get_installer
+except ImportError:
+    get_installer = None
+
+
+class TestSOLRPush(unittest.TestCase):
+    """Test that rer.solrpush is properly installed."""
+
+    layer = RER_SOLRPUSH_API_FUNCTIONAL_TESTING
+
+    def setUp(self):
+        """Custom shared utility setup for tests."""
+        self.portal = self.layer["portal"]
+        self.request = self.layer["request"]
+        setRoles(self.portal, TEST_USER_ID, ["Manager"])
+        set_registry_record(
+            "enabled_types",
+            [u"Document", u"File"],
+            interface=IRerSolrpushSettings,
+        )
+
+    def tearDown(self):
+        set_registry_record("active", True, interface=IRerSolrpushSettings)
+        reset_solr()
+
+    def test_item_not_indexed_if_solrpush_is_not_ready(self):
+        solr_url = get_registry_record("solr_url", IRerSolrpushSettings)
+        api.content.create(container=self.portal, type="Document", title="foo")
+        commit()
+        res = requests.get("{}/select?q=*%3A*&wt=json".format(solr_url)).json()
+        self.assertEqual(res["response"]["numFound"], 0)
+
+        init_solr_push()
+
+        api.content.create(container=self.portal, type="Document", title="bar")
+        commit()
+        res = requests.get("{}/select?q=*%3A*&wt=json".format(solr_url)).json()
+
+        # because initial state id private
+        self.assertEqual(res["response"]["numFound"], 0)
+
+        # File types has no wf, so they are published
+        api.content.create(
+            container=self.portal, type="File", title="bar file"
+        )
+        commit()
+        res = requests.get("{}/select?q=*%3A*&wt=json".format(solr_url)).json()
+        self.assertEqual(res["response"]["numFound"], 1)
+
+
+class TestEvents(unittest.TestCase):
+    """Test that rer.solrpush is properly installed."""
+
+    layer = RER_SOLRPUSH_API_FUNCTIONAL_TESTING
+
+    def setUp(self):
+        """Custom shared utility setup for tests."""
+        self.portal = self.layer["portal"]
+        self.request = self.layer["request"]
+        setRoles(self.portal, TEST_USER_ID, ["Manager"])
+        set_registry_record(
+            "enabled_types",
+            [u"Document", u"File"],
+            interface=IRerSolrpushSettings,
+        )
+        init_solr_push()
+
+    def tearDown(self):
+        set_registry_record("active", True, interface=IRerSolrpushSettings)
+        reset_solr()
+
+    def clean_solr(self):
+        solr_url = get_registry_record("solr_url", IRerSolrpushSettings)
+        solr_clean_url = "{0}/update?stream.body=<delete><query>*:*</query></delete>&commit=true".format(  # noqa
+            solr_url
+        )
+        requests.get(solr_clean_url)
+
+    def get_solr_results(self):
+        solr_url = get_registry_record("solr_url", IRerSolrpushSettings)
+        return requests.get(
+            "{}/select?q=*%3A*&wt=json".format(solr_url)
+        ).json()
+
+    def create_indexed_doc(self):
+        published_doc = api.content.create(
+            container=self.portal, type="Document", title="foo"
+        )
+        api.content.transition(obj=published_doc, transition="publish")
+        commit()
+        return published_doc
+
+    def test_create_indexeable_content(self):
+        api.content.create(container=self.portal, type="Document", title="foo")
+        commit()
+        res = self.get_solr_results()
+        self.assertEqual(res["response"]["numFound"], 0)
+
+        api.content.create(container=self.portal, type="File", title="foo")
+        commit()
+        res = self.get_solr_results()
+        self.assertEqual(res["response"]["numFound"], 1)
+
+    def test_create_non_indexeable_content(self):
+        # we use images because hte don't have a workflow, like Files.
+        api.content.create(container=self.portal, type="Image", title="bara")
+        commit()
+        res = self.get_solr_results()
+        self.assertEqual(res["response"]["numFound"], 0)
+
+    def test_edit_content(self):
+        doc = self.create_indexed_doc()
+        res = self.get_solr_results()
+        self.assertEqual(res["response"]["numFound"], 1)
+        res_doc = res["response"]["docs"][0]
+        self.assertEqual("foo", res_doc.get("Title", ""))
+        self.assertEqual(res_doc.get("Description", ""), "")
+
+        doc.setDescription("foo description")
+        notify(ObjectModifiedEvent(doc))
+        commit()
+
+        res = self.get_solr_results()
+        res_doc = res["response"]["docs"][0]
+        self.assertEqual("foo description", res_doc.get("Description", ""))
+
+    def test_change_state(self):
+        doc = self.create_indexed_doc()
+        res = self.get_solr_results()
+        self.assertEqual(res["response"]["numFound"], 1)
+        res_doc = res["response"]["docs"][0]
+        self.assertEqual("published", res_doc.get("review_state", ""))
+        api.content.transition(obj=doc, transition="retract")
+        commit()
+
+        res = self.get_solr_results()
+        self.assertEqual(res["response"]["numFound"], 0)
+
+    def test_rename_content(self):
+        doc = self.create_indexed_doc()
+        res = self.get_solr_results()
+        self.assertEqual(res["response"]["numFound"], 1)
+        res_doc = res["response"]["docs"][0]
+        self.assertEqual("foo", res_doc.get("id", ""))
+        api.content.rename(obj=doc, new_id="bar")
+        commit()
+
+        res = self.get_solr_results()
+        res_doc = res["response"]["docs"][0]
+        self.assertEqual("bar", res_doc.get("id", ""))
+
+    def test_delete_content(self):
+        doc = self.create_indexed_doc()
+        res = self.get_solr_results()
+        self.assertEqual(res["response"]["numFound"], 1)
+        api.content.delete(obj=doc)
+        commit()
+
+        res = self.get_solr_results()
+        self.assertEqual(res["response"]["numFound"], 0)

rer/solrpush/tests/test_image_tags.py

@@ -0,0 +1,85 @@
+# -*- coding: utf-8 -*-
+"""Setup tests for this package."""
+from plone import api
+from plone.api.portal import set_registry_record
+from plone.app.testing import setRoles
+from plone.app.testing import TEST_USER_ID
+from plone.namedfile.file import NamedFile
+from rer.solrpush.interfaces.settings import IRerSolrpushSettings
+from rer.solrpush.utils import init_solr_push
+from rer.solrpush.utils import reset_solr
+from rer.solrpush.utils import search
+from rer.solrpush.testing import RER_SOLRPUSH_API_FUNCTIONAL_TESTING
+from transaction import commit
+
+import unittest
+import os
+
+
+class TestTika(unittest.TestCase):
+    """Test solr indexing for files"""
+
+    layer = RER_SOLRPUSH_API_FUNCTIONAL_TESTING
+
+    def setUp(self):
+        """Custom shared utility setup for tests."""
+        self.portal = self.layer["portal"]
+        self.request = self.layer["request"]
+        setRoles(self.portal, TEST_USER_ID, ["Manager"])
+        set_registry_record(
+            "enabled_types", [u"File"], interface=IRerSolrpushSettings
+        )
+        init_solr_push()
+        commit()
+
+    def tearDown(self):
+        set_registry_record("active", True, interface=IRerSolrpushSettings)
+        reset_solr()
+        commit()
+
+    def create_file_item(self, filename):
+        item_file = None
+        with open(
+            os.path.join(os.path.dirname(__file__), "docs", filename), "rb"
+        ) as f:
+            item_file = NamedFile(data=f.read(), filename=filename)
+
+        file_item = api.content.create(
+            container=self.portal, type="File", title=filename, file=item_file
+        )
+        commit()
+        return file_item
+
+    def test_index_and_extract_pdf(self):
+        file_item = self.create_file_item(u"example.pdf")
+
+        solr_results = search(query={"*": "*"}, fl="UID")
+        self.assertEqual(solr_results.hits, 1)
+        st_results = search(query={"SearchablesText": "lorem"}, fl="UID")
+        self.assertEqual(st_results.hits, 1)
+        self.assertEqual(file_item.UID(), st_results.docs[0]["UID"])
+
+    def test_index_and_extract_docx(self):
+        file_item = self.create_file_item(u"example.docx")
+
+        solr_results = search(query={"*": "*"}, fl="UID SearchableText")
+        self.assertEqual(solr_results.hits, 1)
+        st_results = search(query={"SearchableText": "ipsum"}, fl="UID")
+        self.assertEqual(st_results.hits, 1)
+        self.assertEqual(file_item.UID(), st_results.docs[0]["UID"])
+
+    def test_index_and_extract_ods(self):
+        file_item = self.create_file_item(u"example.ods")
+
+        solr_results = search(query={"*": "*"}, fl="UID SearchableText")
+        self.assertEqual(solr_results.hits, 1)
+        st_results = search(query={"SearchableText": "yyy"}, fl="UID")
+        self.assertEqual(st_results.hits, 1)
+        self.assertEqual(file_item.UID(), st_results.docs[0]["UID"])
+
+    def test_indexed_file_has_right_mimetype(self):
+        self.create_file_item(u"example.pdf")
+
+        solr_results = search(query={"*": "*"}, fl="UID mime_type")
+        self.assertEqual(solr_results.hits, 1)
+        self.assertEqual(solr_results.docs[0]["mime_type"], "application/pdf")