modoboa 2.4.10__py3-none-any.whl → 2.5.0__py3-none-any.whl

modoboa/amavis/factories.py

@@ -0,0 +1,164 @@
+"""Amavis factories."""
+
+import datetime
+import time
+
+import factory
+
+from . import models
+from .utils import smart_bytes
+
+SPAM_BODY = """X-Envelope-To: <{rcpt}>
+X-Envelope-To-Blocked: <{rcpt}>
+X-Quarantine-ID: <nq6ekd4wtXZg>
+X-Spam-Flag: YES
+X-Spam-Score: 1000.985
+X-Spam-Level: ****************************************************************
+X-Spam-Status: Yes, score=1000.985 tag=2 tag2=6.31 kill=6.31
+    tests=[ALL_TRUSTED=-1, GTUBE=1000, PYZOR_CHECK=1.985]
+    autolearn=no autolearn_force=no
+Received: from demo.modoboa.org ([127.0.0.1])
+    by localhost (demo.modoboa.org [127.0.0.1]) (amavisd-new, port 10024)
+    with ESMTP id nq6ekd4wtXZg for <user@demo.local>;
+    Thu,  9 Nov 2017 15:59:52 +0100 (CET)
+Received: from demo.modoboa.org (localhost [127.0.0.1])
+    by demo.modoboa.org (Postfix) with ESMTP
+    for <user@demo.local>; Thu,  9 Nov 2017 15:59:52 +0100 (CET)
+Content-Type: text/plain; charset="utf-8"
+MIME-Version: 1.0
+Content-Transfer-Encoding: quoted-printable
+Subject: Sample message
+From: {sender}
+To: {rcpt}
+Message-ID: <151023959268.5550.5713670714483771838@demo.modoboa.org>
+Date: Thu, 09 Nov 2017 15:59:52 +0100
+
+This is the GTUBE, the
+        Generic
+        Test for
+        Unsolicited
+        Bulk
+        Email
+
+If your spam filter supports it, the GTUBE provides a test by which you
+can verify that the filter is installed correctly and is detecting incoming
+spam. You can send yourself a test mail containing the following string of
+characters (in upper case and with no white spaces and line breaks):
+
+XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X
+
+You should send this test mail from an account outside of your network.
+"""
+
+VIRUS_BODY = r"""Subject: Virus Test Message (EICAR)
+MIME-Version: 1.0
+Content-Type: multipart/mixed; boundary="huq684BweRXVnRxX"
+Content-Disposition: inline
+Date: Sun, 06 Nov 2011 10:08:18 -0800
+
+
+--huq684BweRXVnRxX
+Content-Type: text/plain; charset=us-ascii
+Content-Disposition: inline
+
+
+This is a virus test message. It contains an attached file 'eicar.com',
+which contains the EICAR virus <http://eicar.org/86-0-Intended-use.html>
+test pattern.
+
+
+--huq684BweRXVnRxX
+Content-Type: application/x-msdos-program
+Content-Disposition: attachment; filename="eicar.com"
+Content-Transfer-Encoding: quoted-printable
+
+X5O!P%@AP[4\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*=0A
+--huq684BweRXVnRxX--
+"""
+
+
+class MaddrFactory(factory.django.DjangoModelFactory):
+    """Factory for Maddr."""
+
+    class Meta:
+        model = models.Maddr
+        django_get_or_create = ("email",)
+
+    id = factory.Sequence(lambda n: n)  # NOQA:A003
+    email = factory.Sequence(lambda n: f"user_{n}@domain.test")
+    domain = "test.domain"
+
+
+class MsgsFactory(factory.django.DjangoModelFactory):
+    """Factory for Mailaddr."""
+
+    class Meta:
+        model = models.Msgs
+
+    mail_id = factory.Sequence(lambda n: f"mailid{n}".encode("ascii"))
+    secret_id = factory.Sequence(lambda n: smart_bytes(f"id{n}"))
+    sid = factory.SubFactory(MaddrFactory)
+    client_addr = "127.0.0.1"
+    originating = "Y"
+    dsn_sent = "N"
+    subject = factory.Sequence(lambda n: f"Test message {n}")
+    time_num = factory.LazyAttribute(lambda o: int(time.time()))
+    time_iso = factory.LazyAttribute(
+        lambda o: datetime.datetime.fromtimestamp(o.time_num).isoformat()
+    )
+    size = 100
+
+
+class MsgrcptFactory(factory.django.DjangoModelFactory):
+    """Factory for Msgrcpt."""
+
+    class Meta:
+        model = models.Msgrcpt
+
+    rseqnum = 1
+    is_local = "Y"
+    bl = "N"
+    wl = "N"
+    mail = factory.SubFactory(MsgsFactory)
+    rid = factory.SubFactory(MaddrFactory)
+
+
+class QuarantineFactory(factory.django.DjangoModelFactory):
+    """Factory for Quarantine."""
+
+    class Meta:
+        model = models.Quarantine
+
+    chunk_ind = 1
+    mail = factory.SubFactory(MsgsFactory)
+
+
+def create_quarantined_msg(rcpt, sender, rs, body, **kwargs):
+    """Create a quarantined msg."""
+    msgrcpt = MsgrcptFactory(
+        rs=rs,
+        rid__email=rcpt,
+        rid__domain="com.test",  # FIXME
+        mail__sid__email=smart_bytes(sender),
+        mail__sid__domain="",  # FIXME
+        **kwargs,
+    )
+    QuarantineFactory(
+        mail=msgrcpt.mail,
+        mail_text=smart_bytes(SPAM_BODY.format(rcpt=rcpt, sender=sender)),
+    )
+    return msgrcpt
+
+
+def create_spam(rcpt, sender="spam@evil.corp", rs=" "):
+    """Create a spam."""
+    body = SPAM_BODY.format(rcpt=rcpt, sender=sender)
+    body += "fóó bár"
+    return create_quarantined_msg(
+        rcpt, sender, rs, body, bspam_level=999.0, content="S"
+    )
+
+
+def create_virus(rcpt, sender="virus@evil.corp", rs=" "):
+    """Create a virus."""
+    return create_quarantined_msg(rcpt, sender, rs, VIRUS_BODY, content="V")

modoboa/amavis/handlers.py

@@ -0,0 +1,146 @@
+"""Amavis handlers."""
+
+from django.db.models import signals
+from django.dispatch import receiver
+from django.utils.translation import gettext as _
+
+from modoboa.admin import models as admin_models
+from modoboa.core import signals as core_signals
+from modoboa.lib import signals as lib_signals
+from modoboa.parameters import tools as param_tools
+from .lib import (
+    create_user_and_policy,
+    create_user_and_use_policy,
+    delete_user,
+    delete_user_and_policy,
+    update_user_and_policy,
+)
+from .models import Policy, Users
+from .sql_connector import SQLconnector
+
+
+@receiver(signals.post_save, sender=admin_models.Domain)
+def manage_domain_policy(sender, instance, **kwargs):
+    """Create user and policy when a domain is added."""
+    if kwargs.get("created"):
+        create_user_and_policy(f"@{instance.name}")
+    else:
+        update_user_and_policy(f"@{instance.oldname}", f"@{instance.name}")
+
+
+@receiver(signals.pre_delete, sender=admin_models.Domain)
+def on_domain_deleted(sender, instance, **kwargs):
+    """Delete user and policy for domain."""
+    delete_user_and_policy(f"@{instance.name}")
+
+
+@receiver(signals.post_save, sender=admin_models.DomainAlias)
+def on_domain_alias_created(sender, instance, **kwargs):
+    """Create user and use domain policy for domain alias."""
+    if not kwargs.get("created"):
+        return
+    create_user_and_use_policy(f"@{instance.name}", f"@{instance.target.name}")
+
+
+@receiver(signals.pre_delete, sender=admin_models.DomainAlias)
+def on_domain_alias_deleted(sender, instance, **kwargs):
+    """Delete user for domain alias."""
+    delete_user(f"@{instance.name}")
+
+
+@receiver(signals.post_save, sender=admin_models.Mailbox)
+def on_mailbox_modified(sender, instance, **kwargs):
+    """Update amavis records if address has changed."""
+    condition = (
+        not param_tools.get_global_parameter("manual_learning")
+        or not hasattr(instance, "old_full_address")
+        or instance.full_address == instance.old_full_address
+    )
+    if condition:
+        return
+    try:
+        user = Users.objects.select_related("policy").get(
+            email=instance.old_full_address
+        )
+    except Users.DoesNotExist:
+        return
+    full_address = instance.full_address
+    user.email = full_address
+    user.policy.policy_name = full_address[:32]
+    user.policy.sa_username = full_address
+    user.policy.save()
+    user.save()
+
+
+@receiver(signals.pre_delete, sender=admin_models.Mailbox)
+def on_mailbox_deleted(sender, instance, **kwargs):
+    """Clean amavis database when a mailbox is removed."""
+    if not param_tools.get_global_parameter("manual_learning"):
+        return
+    delete_user_and_policy(f"@{instance.full_address}")
+
+
+@receiver(signals.post_save, sender=admin_models.AliasRecipient)
+def on_aliasrecipient_created(sender, instance, **kwargs):
+    """Create amavis record for the new alias recipient.
+
+    FIXME: how to deal with distibution lists ?
+    """
+    conf = dict(param_tools.get_global_parameters("amavis"))
+    condition = (
+        not conf["manual_learning"]
+        or not conf["user_level_learning"]
+        or not instance.r_mailbox
+        or instance.alias.type != "alias"
+    )
+    if condition:
+        return
+    policy = Policy.objects.filter(policy_name=instance.r_mailbox.full_address).first()
+    if policy:
+        # Use mailbox policy for this new alias. We update or create
+        # to handle the case where an account is being replaced by an
+        # alias (when it is disabled).
+        email = instance.alias.address
+        Users.objects.update_or_create(
+            email=email, defaults={"policy": policy, "fullname": email, "priority": 7}
+        )
+
+
+@receiver(signals.pre_delete, sender=admin_models.Alias)
+def on_mailboxalias_deleted(sender, instance, **kwargs):
+    """Clean amavis database when an alias is removed."""
+    if not param_tools.get_global_parameter("manual_learning"):
+        return
+    if instance.address.startswith("@"):
+        # Catchall alias, do not remove domain entry accidentally...
+        return
+    aliases = [instance.address]
+    Users.objects.filter(email__in=aliases).delete()
+
+
+@receiver(core_signals.get_top_notifications)
+def check_for_pending_requests(sender, include_all, **kwargs):
+    """Check if release requests are pending."""
+    request = lib_signals.get_request()
+    condition = (
+        param_tools.get_global_parameter("user_can_release")
+        or request.user.role == "SimpleUsers"
+    )
+    if condition:
+        return []
+
+    nbrequests = SQLconnector(user=request.user).get_pending_requests()
+    if not nbrequests:
+        return [{"id": "nbrequests", "counter": 0}] if include_all else []
+
+    url = "/user/quarantine?requests=1"
+    return [
+        {
+            "id": "nbrequests",
+            "url": url,
+            "text": _("Pending requests"),
+            "counter": nbrequests,
+            "color": "error",
+            "target": "all",
+        }
+    ]

modoboa/amavis/lib.py

@@ -0,0 +1,381 @@
+import os
+import re
+import socket
+import struct
+from email.utils import parseaddr
+
+import idna
+
+from django.conf import settings
+from django.utils.translation import gettext as _
+
+from rest_framework import authentication, exceptions
+
+from modoboa.admin import models as admin_models
+from modoboa.lib.email_utils import split_address, split_local_part, split_mailbox
+from modoboa.lib.exceptions import InternalError
+from modoboa.lib.sysutils import exec_cmd
+from modoboa.parameters import tools as param_tools
+from .models import Msgrcpt, Policy, Users
+from .utils import smart_bytes, smart_str
+
+
+class SelfServiceAuthentication(authentication.BaseAuthentication):
+
+    def authenticate(self, request):
+        from .sql_connector import SQLconnector
+
+        mail_id = request._request.resolver_match.kwargs.get("pk")
+        if request.method == "GET":
+            rcpt = request.GET.get("rcpt")
+            secret_id = request.GET.get("secret_id")
+        else:
+            rcpt = request.data.get("rcpt")
+            secret_id = request.data.get("secret_id")
+        if not mail_id or not rcpt or not secret_id:
+            return None
+        connector = SQLconnector()
+        try:
+            msgrcpt = connector.get_recipient_message(rcpt, mail_id)
+        except Msgrcpt.DoesNotExist:
+            raise exceptions.AuthenticationFailed("Invalid credentials") from None
+        if secret_id != smart_str(msgrcpt.mail.secret_id):
+            raise exceptions.AuthenticationFailed("Invalid credentials")
+        return (None, "selfservice")
+
+
+class AMrelease:
+    def __init__(self):
+        conf = dict(param_tools.get_global_parameters("amavis"))
+        try:
+            if conf["am_pdp_mode"] == "inet":
+                self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+                self.sock.connect((conf["am_pdp_host"], conf["am_pdp_port"]))
+            else:
+                self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+                self.sock.connect(conf["am_pdp_socket"])
+        except OSError as err:
+            raise InternalError(
+                _("Connection to amavis failed: %s") % str(err)
+            ) from None
+
+    def decode(self, answer):
+        def repl(match):
+            return struct.pack("B", int(match.group(0)[1:], 16))
+
+        return re.sub(rb"%([0-9a-fA-F]{2})", repl, answer)
+
+    def __del__(self):
+        self.sock.close()
+
+    def sendreq(self, mailid, secretid, recipient, *others):
+        self.sock.send(
+            smart_bytes(
+                f"""request=release
+mail_id={smart_str(mailid)}
+secret_id={smart_str(secretid)}
+quar_type=Q
+recipient={smart_str(recipient)}
+
+"""
+            )
+        )
+        answer = self.sock.recv(1024)
+        answer = self.decode(answer)
+        if re.search(rb"250 [\d\.]+ Ok", answer):
+            return True
+        return False
+
+
+class SpamassassinClient:
+    """A stupid spamassassin client."""
+
+    def __init__(self, user, recipient_db):
+        """Constructor."""
+        conf = dict(param_tools.get_global_parameters("amavis"))
+        self._sa_is_local = conf["sa_is_local"]
+        self._default_username = conf["default_user"]
+        self._recipient_db = recipient_db
+        self._setup_cache = {}
+        self._username_cache = []
+        if user.role == "SimpleUsers":
+            if conf["user_level_learning"]:
+                self._username = user.email
+        else:
+            self._username = None
+        self.error = None
+        if self._sa_is_local:
+            self._learn_cmd = self._find_binary("sa-learn")
+            self._learn_cmd += " --{0} --no-sync -u {1}"
+            self._learn_cmd_kwargs = {}
+            self._expected_exit_codes = [0]
+            self._sync_cmd = self._find_binary("sa-learn")
+            self._sync_cmd += " -u {0} --sync"
+        else:
+            self._learn_cmd = self._find_binary("spamc")
+            self._learn_cmd += " -d {} -p {}".format(
+                conf["spamd_address"], conf["spamd_port"]
+            )
+            self._learn_cmd += " -L {0} -u {1}"
+            self._learn_cmd_kwargs = {}
+            self._expected_exit_codes = [5, 6]
+
+    def _find_binary(self, name):
+        """Find path to binary."""
+        code, output = exec_cmd(f"which {name}")
+        if not code:
+            return smart_str(output).strip()
+        known_paths = getattr(settings, "SA_LOOKUP_PATH", ("/usr/bin",))
+        for path in known_paths:
+            bpath = os.path.join(path, name)
+            if os.path.isfile(bpath) and os.access(bpath, os.X_OK):
+                return bpath
+        raise InternalError(_("Failed to find {} binary").format(name))
+
+    def _get_mailbox_from_rcpt(self, rcpt):
+        """Retrieve a mailbox from a recipient address."""
+        local_part, domname, extension = split_mailbox(rcpt, return_extension=True)
+        try:
+            mailbox = admin_models.Mailbox.objects.select_related("domain").get(
+                address=local_part, domain__name=domname
+            )
+        except admin_models.Mailbox.DoesNotExist:
+            alias = admin_models.Alias.objects.filter(
+                address=f"{local_part}@{domname}",
+                aliasrecipient__r_mailbox__isnull=False,
+            ).first()
+            if not alias:
+                raise InternalError(_("No recipient found")) from None
+            if alias.type != "alias":
+                return None
+            mailbox = alias.aliasrecipient_set.filter(r_mailbox__isnull=False).first()
+        return mailbox
+
+    def _get_domain_from_rcpt(self, rcpt):
+        """Retrieve a domain from a recipient address."""
+        local_part, domname = split_mailbox(rcpt)
+        domain = admin_models.Domain.objects.filter(name=domname).first()
+        if not domain:
+            raise InternalError(_("Local domain not found"))
+        return domain
+
+    def _learn(self, rcpt, msg, mtype):
+        """Internal method to call the learning command."""
+        if self._username is None:
+            if self._recipient_db == "global":
+                username = self._default_username
+            elif self._recipient_db == "domain":
+                domain = self._get_domain_from_rcpt(rcpt)
+                username = domain.name
+                condition = (
+                    username not in self._setup_cache
+                    and setup_manual_learning_for_domain(domain)
+                )
+                if condition:
+                    self._setup_cache[username] = True
+            else:
+                mbox = self._get_mailbox_from_rcpt(rcpt)
+                if mbox is None:
+                    username = self._default_username
+                else:
+                    if isinstance(mbox, admin_models.Mailbox):
+                        username = mbox.full_address
+                    elif isinstance(mbox, admin_models.AliasRecipient):
+                        username = mbox.address
+                    else:
+                        username = None
+                    condition = (
+                        username is not None
+                        and username not in self._setup_cache
+                        and setup_manual_learning_for_mbox(mbox)
+                    )
+                    if condition:
+                        self._setup_cache[username] = True
+        else:
+            username = self._username
+            if username not in self._setup_cache:
+                mbox = self._get_mailbox_from_rcpt(username)
+                if mbox and setup_manual_learning_for_mbox(mbox):
+                    self._setup_cache[username] = True
+        if username not in self._username_cache:
+            self._username_cache.append(username)
+        cmd = self._learn_cmd.format(mtype, username)
+        code, output = exec_cmd(cmd, pinput=smart_bytes(msg), **self._learn_cmd_kwargs)
+        if code in self._expected_exit_codes:
+            return True
+        self.error = smart_str(output)
+        return False
+
+    def learn_spam(self, rcpt, msg):
+        """Learn new spam."""
+        return self._learn(rcpt, msg, "spam")
+
+    def learn_ham(self, rcpt, msg):
+        """Learn new ham."""
+        return self._learn(rcpt, msg, "ham")
+
+    def done(self):
+        """Call this method at the end of the processing."""
+        if self._sa_is_local:
+            for username in self._username_cache:
+                cmd = self._sync_cmd.format(username)
+                exec_cmd(cmd, **self._learn_cmd_kwargs)
+
+
+def create_user_and_policy(name, priority=7):
+    """Create records.
+
+    Create two records (a user and a policy) using :keyword:`name` as
+    an identifier.
+
+    :param str name: name
+    :return: the new ``Policy`` object
+    """
+    policy, _ = Policy.objects.get_or_create(policy_name=name[:32])
+    if not Users.objects.filter(email=name).exists():
+        Users.objects.create(
+            email=name, fullname=name, priority=priority, policy=policy
+        )
+    return policy
+
+
+def create_user_and_use_policy(name, policy, priority=7):
+    """Create a *users* record and use an existing policy.
+
+    :param str name: user record name
+    :param str policy: string or Policy instance
+    """
+    if isinstance(policy, str):
+        policy = Policy.objects.get(policy_name=policy[:32])
+    Users.objects.get_or_create(
+        email=name, fullname=name, priority=priority, policy=policy
+    )
+
+
+def update_user_and_policy(oldname, newname):
+    """Update records.
+
+    :param str oldname: old name
+    :param str newname: new name
+    """
+    if oldname == newname:
+        return
+    u = Users.objects.get(email=oldname)
+    u.email = newname
+    u.fullname = newname
+    u.policy.policy_name = newname[:32]
+    u.policy.save(update_fields=["policy_name"])
+    u.save()
+
+
+def delete_user_and_policy(name):
+    """Delete records.
+
+    :param str name: identifier
+    """
+    try:
+        u = Users.objects.get(email=name)
+    except Users.DoesNotExist:
+        return
+    u.policy.delete()
+    u.delete()
+
+
+def delete_user(name):
+    """Delete a *users* record.
+
+    :param str name: user record name
+    """
+    try:
+        Users.objects.get(email=name).delete()
+    except Users.DoesNotExist:
+        pass
+
+
+def manual_learning_enabled(user):
+    """Check if manual learning is enabled or not.
+
+    Also check for :kw:`user` if necessary.
+
+    :return: True if learning is enabled, False otherwise.
+    """
+    conf = dict(param_tools.get_global_parameters("amavis"))
+    if not conf["manual_learning"]:
+        return False
+    if user.role != "SuperAdmins":
+        if user.has_perm("admin.view_domain"):
+            manual_learning = (
+                conf["domain_level_learning"] or conf["user_level_learning"]
+            )
+        else:
+            manual_learning = conf["user_level_learning"]
+        return manual_learning
+    return True
+
+
+def setup_manual_learning_for_domain(domain):
+    """Setup manual learning if necessary.
+
+    :return: True if learning has been setup, False otherwise
+    """
+    if Policy.objects.filter(sa_username=domain.name).exists():
+        return False
+    policy = Policy.objects.get(policy_name=f"@{domain.name[:32]}")
+    policy.sa_username = domain.name
+    policy.save()
+    return True
+
+
+def setup_manual_learning_for_mbox(mbox):
+    """Setup manual learning if necessary.
+
+    :return: True if learning has been setup, False otherwise
+    """
+    result = False
+    if isinstance(mbox, admin_models.AliasRecipient) and mbox.r_mailbox is not None:
+        mbox = mbox.r_mailbox
+    if isinstance(mbox, admin_models.Mailbox):
+        pname = mbox.full_address[:32]
+        if not Policy.objects.filter(policy_name=pname).exists():
+            policy = create_user_and_policy(pname)
+            policy.sa_username = mbox.full_address
+            policy.save()
+            for alias in mbox.alias_addresses:
+                create_user_and_use_policy(alias, policy)
+            result = True
+    return result
+
+
+def make_query_args(address, exact_extension=True, wildcard=None, domain_search=False):
+    assert isinstance(address, str), "address should be of type str"
+    conf = dict(param_tools.get_global_parameters("amavis"))
+    local_part, domain = split_address(address)
+    if not conf["localpart_is_case_sensitive"]:
+        local_part = local_part.lower()
+    if domain:
+        domain = domain.lstrip("@").rstrip(".")
+        domain = domain.lower()
+        orig_domain = domain
+        domain = idna.encode(domain, uts46=True).decode("ascii")
+    delimiter = conf["recipient_delimiter"]
+    local_part, extension = split_local_part(local_part, delimiter=delimiter)
+    query_args = []
+    if conf["localpart_is_case_sensitive"] or (domain and domain != orig_domain):
+        query_args.append(address)
+    if extension:
+        query_args.append(f"{local_part}{delimiter}{extension}@{domain}")
+    if delimiter and not exact_extension and wildcard:
+        query_args.append(f"{local_part}{delimiter}{wildcard}@{domain}")
+    query_args.append(f"{local_part}@{domain}")
+    if domain_search:
+        query_args.append(f"@{domain}")
+        query_args.append("@.")
+
+    return query_args
+
+
+def cleanup_email_address(address):
+    address = parseaddr(address)
+    if address[0]:
+        return f"{address[0]} <{address[1]}>"
+    return address[1]