modoboa 2.4.10__py3-none-any.whl → 2.5.0__py3-none-any.whl

modoboa/amavis/tests/test_viewsets.py

@@ -0,0 +1,319 @@
+import os
+from unittest import mock
+
+from rq import SimpleWorker
+
+from django.core import mail
+from django.core.management import call_command
+from django.test import override_settings
+from django.urls import reverse
+
+import django_rq
+
+from modoboa.admin import factories as admin_factories, models as admin_models
+from modoboa.amavis import factories, models
+from modoboa.amavis.utils import smart_str
+from modoboa.core import models as core_models
+from modoboa.lib.tests import ModoAPITestCase
+
+
+AMAVIS_SETTINGS = {
+    "localpart_is_case_sensitive": False,
+    "recipient_delimiter": "+",
+    "max_attachment_size": 14,
+    "released_msgs_cleanup": False,
+    "am_pdp_mode": "unix",
+    "am_pdp_host": "localhost",
+    "am_pdp_port": 9998,
+    "user_can_release": False,
+    "self_service": False,
+    "manual_learning": True,
+    "sa_is_local": True,
+}
+
+
+class TestDataMixin:
+    """A mixin to provide test data."""
+
+    @classmethod
+    def setUpTestData(cls):  # NOQA:N802
+        """Create some content."""
+        super().setUpTestData()
+        cls.msgrcpt = factories.create_spam("user@test.com")
+
+
+@override_settings(SA_LOOKUP_PATH=(os.path.dirname(__file__),))
+class QuarantineViewSetTestCase(TestDataMixin, ModoAPITestCase):
+
+    databases = "__all__"
+
+    @classmethod
+    def setUpTestData(cls):  # NOQA:N802
+        """Create test data."""
+        super().setUpTestData()
+        admin_factories.populate_database()
+
+    def tearDown(self):
+        """Restore msgrcpt state."""
+        self.msgrcpt.rs = " "
+        self.msgrcpt.save(update_fields=["rs"])
+        self.set_global_parameter("domain_level_learning", False)
+        self.set_global_parameter("user_level_learning", False)
+
+    def test_list(self):
+        url = reverse("v2:amavis-quarantine-list")
+        resp = self.client.get(url)
+        self.assertEqual(resp.status_code, 200)
+        content = resp.json()
+        self.assertEqual(content["count"], 1)
+
+        resp = self.client.get(f"{url}?search=pouet&criteria=both")
+        content = resp.json()
+        self.assertEqual(content["count"], 0)
+
+        resp = self.client.get(f"{url}?msgtype=V")
+        content = resp.json()
+        self.assertEqual(content["count"], 0)
+        factories.create_virus("user@test.com")
+        resp = self.client.get(f"{url}?msgtype=V")
+        content = resp.json()
+        self.assertEqual(content["count"], 1)
+
+    def test_retrieve(self):
+        mail_id = smart_str(self.msgrcpt.mail.mail_id)
+        rcpt = smart_str(self.msgrcpt.rid.email)
+        url = reverse("v2:amavis-quarantine-detail", args=[mail_id])
+        resp = self.client.get(f"{url}?rcpt={rcpt}")
+        self.assertEqual(resp.status_code, 200)
+
+        user = core_models.User.objects.get(username="user@test.com")
+        self.client.force_authenticate(user)
+        resp = self.client.get(f"{url}?rcpt={rcpt}")
+        self.assertEqual(resp.status_code, 200)
+        self.msgrcpt.refresh_from_db()
+        self.assertEqual(self.msgrcpt.rs, "V")
+
+    def test_headers(self):
+        mail_id = smart_str(self.msgrcpt.mail.mail_id)
+        url = reverse("v2:amavis-quarantine-headers", args=[mail_id])
+        resp = self.client.get(url)
+        self.assertEqual(resp.status_code, 200)
+
+    def test_headers_selfservice(self):
+        self.client.logout()
+        mail_id = smart_str(self.msgrcpt.mail.mail_id)
+        url = reverse("v2:amavis-quarantine-headers", args=[mail_id])
+        url = f"{url}?secret_id={smart_str(self.msgrcpt.mail.secret_id)}"
+        response = self.client.get(url)
+        self.assertEqual(response.status_code, 403)
+
+        self.set_global_parameter("self_service", True)
+        response = self.client.get(url)
+        self.assertEqual(response.status_code, 403)
+        url = f"{url}&rcpt={smart_str(self.msgrcpt.rid.email)}"
+        response = self.client.get(url)
+        self.assertEqual(response.status_code, 200)
+
+    def test_delete(self):
+        mail_id = smart_str(self.msgrcpt.mail.mail_id)
+        url = reverse("v2:amavis-quarantine-delete", args=[mail_id])
+        data = {"mailid": mail_id, "rcpt": smart_str(self.msgrcpt.rid.email)}
+        response = self.client.post(url, data, format="json")
+        self.assertEqual(response.status_code, 204)
+        self.msgrcpt.refresh_from_db()
+        self.assertEqual(self.msgrcpt.rs, "D")
+
+    def test_delete_selfservice(self):
+        self.client.logout()
+        mail_id = smart_str(self.msgrcpt.mail.mail_id)
+        url = reverse("v2:amavis-quarantine-delete", args=[mail_id])
+        data = {"mailid": mail_id, "rcpt": smart_str(self.msgrcpt.rid.email)}
+        response = self.client.post(url, data, format="json")
+        self.assertEqual(response.status_code, 403)
+        self.set_global_parameter("self_service", True)
+        data["secret_id"] = smart_str(self.msgrcpt.mail.secret_id)
+        response = self.client.post(url, data, format="json")
+        self.assertEqual(response.status_code, 204)
+        self.msgrcpt.refresh_from_db()
+        self.assertEqual(self.msgrcpt.rs, "D")
+
+    def test_delete_selection(self):
+        mail_id = smart_str(self.msgrcpt.mail.mail_id)
+        url = reverse("v2:amavis-quarantine-delete-selection")
+        data = {
+            "selection": [
+                {"mailid": mail_id, "rcpt": smart_str(self.msgrcpt.rid.email)}
+            ]
+        }
+        response = self.client.post(url, data, format="json")
+        self.assertEqual(response.status_code, 204)
+        self.msgrcpt.refresh_from_db()
+        self.assertEqual(self.msgrcpt.rs, "D")
+
+    @mock.patch("socket.socket")
+    def test_release(self, mock_socket):
+        mock_socket.return_value.recv.return_value = b"250 1234 Ok\r\n"
+        mail_id = smart_str(self.msgrcpt.mail.mail_id)
+        url = reverse("v2:amavis-quarantine-release", args=[mail_id])
+        data = {"mailid": mail_id, "rcpt": smart_str(self.msgrcpt.rid.email)}
+        response = self.client.post(url, data, format="json")
+        self.assertEqual(response.status_code, 200)
+        self.msgrcpt.refresh_from_db()
+        self.assertEqual(self.msgrcpt.rs, "R")
+
+    @mock.patch("socket.socket")
+    def test_release_selfservice(self, mock_socket):
+        """Test release view."""
+        mock_socket.return_value.recv.return_value = b"250 1234 Ok\r\n"
+        self.client.logout()
+        mail_id = smart_str(self.msgrcpt.mail.mail_id)
+        url = reverse("v2:amavis-quarantine-release", args=[mail_id])
+        self.set_global_parameter("self_service", True)
+        # Missing rcpt -> fails
+        data = {"mailid": mail_id, "secret_id": "1234"}
+        resp = self.client.post(url, data, format="json")
+        self.assertEqual(resp.status_code, 403)
+        # Wrong secret_id -> fails
+        data["rcpt"] = smart_str(self.msgrcpt.rid.email)
+        resp = self.client.post(url, data, format="json")
+        self.assertEqual(resp.status_code, 403)
+        # Request mode
+        data["secret_id"] = smart_str(self.msgrcpt.mail.secret_id)
+        resp = self.client.post(url, data, format="json")
+        self.assertEqual(resp.status_code, 200)
+        self.msgrcpt.refresh_from_db()
+        self.assertEqual(self.msgrcpt.rs, "p")
+        # Direct release mode
+        self.set_global_parameter("user_can_release", True)
+        resp = self.client.post(url, data, format="json")
+        self.assertEqual(resp.status_code, 200)
+        self.msgrcpt.refresh_from_db()
+        self.assertEqual(self.msgrcpt.rs, "R")
+
+    def test_release_request(self):
+        """Test release request mode."""
+        user = core_models.User.objects.get(username="user@test.com")
+        self.client.force_authenticate(user)
+
+        mail_id = smart_str(self.msgrcpt.mail.mail_id)
+        url = reverse("v2:amavis-quarantine-release", args=[mail_id])
+        data = {"mailid": mail_id, "rcpt": smart_str(self.msgrcpt.rid.email)}
+        response = self.client.post(url, data, format="json")
+        self.assertEqual(response.status_code, 200)
+        self.msgrcpt.refresh_from_db()
+        self.assertEqual(self.msgrcpt.rs, "p")
+
+        # Send notification to admins
+        call_command("amnotify")
+        self.assertEqual(len(mail.outbox), 1)
+
+    def _test_mark_selection(self, action, status):
+        """Mark message common code."""
+        mail_id = smart_str(self.msgrcpt.mail.mail_id)
+        url = reverse("v2:amavis-quarantine-mark-selection")
+        data = {
+            "selection": [
+                {"mailid": mail_id, "rcpt": smart_str(self.msgrcpt.rid.email)}
+            ],
+            "type": action,
+        }
+        response = self.client.post(url, data, format="json")
+        self.assertEqual(response.status_code, 204)
+        queue = django_rq.get_queue("modoboa")
+        worker = SimpleWorker([queue], connection=queue.connection)
+        worker.work(burst=True)
+        self.msgrcpt.refresh_from_db()
+        self.assertEqual(self.msgrcpt.rs, status)
+
+        self.msgrcpt.rs = " "
+        self.msgrcpt.save(update_fields=["rs"])
+        self.set_global_parameter("sa_is_local", False)
+        response = self.client.post(url, data, format="json")
+        self.assertEqual(response.status_code, 204)
+        worker.work(burst=True)
+        self.msgrcpt.refresh_from_db()
+        self.assertEqual(self.msgrcpt.rs, status)
+
+    def test_mark_as_ham(self):
+        """Test mark_as_ham view."""
+        self._test_mark_selection("ham", "H")
+
+        # Check domain level learning
+        self.set_global_parameter("domain_level_learning", True)
+        data = {
+            "selection": [
+                {
+                    "rcpt": smart_str(self.msgrcpt.rid.email),
+                    "mailid": smart_str(self.msgrcpt.mail.mail_id),
+                },
+            ],
+            "type": "ham",
+            "database": "domain",
+        }
+        url = reverse("v2:amavis-quarantine-mark-selection")
+        response = self.client.post(url, data, format="json")
+        self.assertEqual(response.status_code, 204)
+
+        # Check user level learning
+        self.set_global_parameter("user_level_learning", True)
+        data["database"] = "user"
+        response = self.client.post(url, data, format="json")
+        self.assertEqual(response.status_code, 204)
+
+    def test_mark_as_spam(self):
+        """Test mark_as_spam view."""
+        self._test_mark_selection("spam", "S")
+
+    def test_manual_learning_as_user(self):
+        """Test learning when connected as a simple user."""
+        user = core_models.User.objects.get(username="user@test.com")
+        self.client.force_authenticate(user)
+        mail_id = smart_str(self.msgrcpt.mail.mail_id)
+        data = {
+            "selection": [
+                {"mailid": mail_id, "rcpt": smart_str(self.msgrcpt.rid.email)}
+            ],
+            "type": "ham",
+        }
+        url = reverse("v2:amavis-quarantine-mark-selection")
+        response = self.client.post(url, data, format="json")
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.json(), {"status": "ok"})
+        self.set_global_parameter("user_level_learning", True)
+        self._test_mark_selection("ham", "H")
+
+
+class PolicyViewSetTestCase(ModoAPITestCase):
+
+    databases = "__all__"
+
+    @classmethod
+    def setUpTestData(cls):  # NOQA:N802
+        """Create test data."""
+        super().setUpTestData()
+        admin_factories.populate_database()
+
+    def test_get_policy(self):
+        domain = admin_models.Domain.objects.get(name="test.com")
+        url = reverse("v2:amavis-policy-detail", args=[domain.pk])
+        resp = self.client.get(url)
+        self.assertEqual(resp.status_code, 200)
+
+    def test_update_policy(self):
+        domain = admin_models.Domain.objects.get(name="test.com")
+        data = {"bypass_virus_checks": "Y", "spam_tag2_level": 6}
+        url = reverse("v2:amavis-policy-detail", args=[domain.pk])
+        resp = self.client.patch(url, data, format="json")
+        self.assertEqual(resp.status_code, 200)
+        p = models.Policy.objects.get(users__email="@test.com")
+        self.assertEqual(p.bypass_virus_checks, data["bypass_virus_checks"])
+        self.assertEqual(p.spam_tag2_level, data["spam_tag2_level"])
+
+
+class ParametersAPITestCase(ModoAPITestCase):
+
+    def test_update(self):
+        settings = AMAVIS_SETTINGS.copy()
+        url = reverse("v2:parameter-global-detail", args=["amavis"])
+        resp = self.client.put(url, settings, format="json")
+        self.assertEqual(resp.status_code, 200)

modoboa/amavis/urls.py

@@ -0,0 +1,11 @@
+"""Amavis urls."""
+
+from rest_framework import routers
+
+from modoboa.amavis import viewsets
+
+router = routers.SimpleRouter()
+router.register(r"quarantine", viewsets.QuarantineViewSet, basename="amavis-quarantine")
+router.register(r"policies", viewsets.PolicyViewSet, basename="amavis-policy")
+
+urlpatterns = router.urls

modoboa/amavis/utils.py

@@ -0,0 +1,105 @@
+"""A collection of utility functions for working with the Amavis database."""
+
+import chardet
+
+from django.conf import settings
+from django.db.models.expressions import Func
+from django.utils.encoding import (
+    smart_bytes as django_smart_bytes,
+    smart_str as django_smart_str,
+)
+
+
+"""
+Byte fields for text data are EVIL.
+
+MySQL uses `varbyte` fields which mysqlclient client maps to `str` (Py2) or
+`bytes` (Py3), Djangos smart_* functions work as expected.
+
+PostgreSQL uses `bytea` fields which psycopg2 maps to `memoryview`,
+Djangos smart_* functions don't work as expected, you must call `tobytes()` on
+the memoryview for them to work.
+
+For convenience use smart_bytes and smart_str from this file in modoboa.amavis
+to avoid any headaches.
+"""
+
+
+def smart_bytes(value, *args, **kwargs):
+    if isinstance(value, memoryview):
+        value = value.tobytes()
+    return django_smart_bytes(value, *args, **kwargs)
+
+
+def smart_str(value, *args, **kwargs):
+    if isinstance(value, memoryview):
+        value = value.tobytes()
+    return django_smart_str(value, *args, **kwargs)
+
+
+def fix_utf8_encoding(value):
+    """Fix utf-8 strings that contain utf-8 escaped characters.
+
+    msgs.from_addr and msgs.subject potentialy contain badly escaped utf-8
+    characters, this utility function fixes that and should be used anytime
+    these fields are accesses.
+
+    Didn't even know the raw_unicode_escape encoding existed :)
+    https://docs.python.org/3/library/codecs.html?highlight=raw_unicode_escape#python-specific-encodings
+    """
+    assert isinstance(value, str), "value should be of type str"
+
+    if len(value) == 0:
+        # short circuit for empty strings
+        return ""
+
+    bytes_value = value.encode("raw_unicode_escape")
+    try:
+        value = bytes_value.decode("utf-8")
+    except UnicodeDecodeError:
+        encoding = chardet.detect(bytes_value)
+        try:
+            value = bytes_value.decode(encoding["encoding"], "replace")
+        except (TypeError, UnicodeDecodeError):
+            # ??? use the original value, we've done our best to try and
+            # convert it to a clean utf-8 string.
+            pass
+
+    return value
+
+
+class ConvertFrom(Func):
+    """Convert a binary value to a string.
+    Calls the database specific function to convert a binary value to a string
+    using the encoding set in AMAVIS_DEFAULT_DATABASE_ENCODING.
+    """
+
+    """PostgreSQL implementation.
+    See https://www.postgresql.org/docs/9.3/static/functions-string.html#FUNCTIONS-STRING-OTHER"""  # NOQA:E501
+    function = "convert_from"
+    arity = 1
+    template = (
+        f"%(function)s(%(expressions)s, '{settings.AMAVIS_DEFAULT_DATABASE_ENCODING}')"
+    )
+
+    def as_mysql(self, compiler, connection):
+        """MySQL implementation.
+        See https://dev.mysql.com/doc/refman/5.5/en/cast-functions.html#function_convert
+        """  # NOQA:E501
+        return super().as_sql(
+            compiler,
+            connection,
+            function="CONVERT",
+            template=f"%(function)s(%(expressions)s USING {settings.AMAVIS_DEFAULT_DATABASE_ENCODING})",
+            arity=1,
+        )
+
+    def as_sqlite(self, compiler, connection):
+        """SQLite implementation.
+        SQLite has no equivilant function, just return the field."""
+        return super().as_sql(
+            compiler,
+            connection,
+            template="%(expressions)s",
+            arity=1,
+        )