udata 7.0.4.dev27568__py2.py3-none-any.whl → 7.0.4.dev27598__py2.py3-none-any.whl

udata/core/discussions/api.py

@@ -14,10 +14,8 @@ from udata.core.user.api_fields import user_ref_fields
 from .forms import DiscussionCreateForm, DiscussionCommentForm
 from .models import Message, Discussion
 from .permissions import CloseDiscussionPermission
-from .signals import (
-    on_new_discussion, on_new_discussion_comment, on_discussion_closed,
-    on_discussion_deleted
-)
+from .signals import on_discussion_deleted
+
 
 ns = api.namespace('discussions', 'Discussion related operations')
 

udata/core/discussions/models.py

@@ -19,6 +19,23 @@ class Message(SpamMixin, db.EmbeddedDocument):
 
     def texts_to_check_for_spam(self):
         return [self.content]
+    
+    def spam_report_message(self, breadcrumb):
+        message =  f"Spam potentiel dans le message"
+        if self.posted_by:
+            message += f" de [{self.posted_by.fullname}]({self.posted_by.external_url})"
+
+        if len(breadcrumb) != 2:
+            log.warning(f"`spam_report_message` called on message with a breadcrumb of {len(breadcrumb)} elements.", extra={ 'breadcrumb': breadcrumb})
+            return message
+        
+        discussion = breadcrumb[0]
+        if not isinstance(discussion, Discussion):
+            log.warning(f"`spam_report_message` called on message with a breadcrumb not containing a Discussion at index 0.", extra={ 'breadcrumb': breadcrumb})
+            return message
+
+        message += f" sur la discussion « [{discussion.title}]({discussion.external_url}) »"
+        return message
 
 
 class Discussion(SpamMixin, db.Document):
@@ -60,11 +77,12 @@ class Discussion(SpamMixin, db.Document):
             _anchor='discussion-{id}'.format(id=self.id),
             _external=True)
     
-    def spam_report_title(self):
-        return self.title
-    
-    def spam_report_link(self):
-        return self.external_url
+    def spam_report_message(self, breadcrumb):
+        message =  f"Spam potentiel sur la discussion « [{self.title}]({self.external_url}) »"
+        if self.user:
+            message += f" de [{self.user.fullname}]({self.user.external_url})"
+
+        return message
     
     @spam_protected()
     def signal_new(self):

udata/core/discussions/tasks.py

@@ -16,8 +16,10 @@ log = get_logger(__name__)
 def owner_recipients(discussion):
     if getattr(discussion.subject, 'organization', None):
         return [m.user for m in discussion.subject.organization.members]
-    else:
+    elif getattr(discussion.subject, 'owner', None):
         return [discussion.subject.owner]
+    else:
+        return []
 
 
 @connect(on_new_discussion, by_id=True)

udata/core/spam/api.py

@@ -53,6 +53,5 @@ class SpamAPI(API):
         discussions = Discussion.objects(Q(spam__status=POTENTIAL_SPAM) | Q(discussion__spam__status=POTENTIAL_SPAM))
 
         return [{
-            'title': discussion.spam_report_title(),
-            'link': discussion.spam_report_link(),
+            'message': discussion.spam_report_message([discussion]),
         } for discussion in discussions]

udata/core/spam/fields.py

@@ -6,7 +6,6 @@ spam_fields = api.model('Spam', {
 })
 
 potential_spam_fields = api.model('PotentialSpam', {
-    'title': fields.String(readonly=True),
-    'link': fields.String(readonly=True),
+    'message': fields.String(readonly=True),
 })
 

udata/core/spam/models.py

@@ -97,7 +97,7 @@ class SpamMixin(object):
 
             # Language detection is not working well with texts of a few words.
             if SpamMixin.allowed_langs() and len(text) > SpamMixin.minimum_string_length_for_lang_check():
-                lang = detect(text)
+                lang = detect(text.lower())
                 if lang not in SpamMixin.allowed_langs():
                     self.spam.status = POTENTIAL_SPAM
                     self._report(text=text, breadcrumb=breadcrumb, reason=f"not allowed language \"{lang}\"")
@@ -143,11 +143,8 @@ class SpamMixin(object):
     def embeds_to_check_for_spam(self):
         return []
 
-    def spam_report_title(self):
-        return type(self).__name__
-
-    def spam_report_link(self):
-        return None
+    def spam_report_message(self):
+        return f"Spam potentiel sur {type(self).__name__}"
 
     def _report(self, text, breadcrumb, reason):
         base_model = breadcrumb[0]
@@ -158,22 +155,14 @@ class SpamMixin(object):
             if document != base_model:
                 return
 
-            # Note that all the chain should be a SpamMixin, maybe we could filter out if it's not the case here…
-            title = " → ".join(map(lambda o: o.spam_report_title(), breadcrumb))
-
-            # Select the first link in the embed list (for example message doesn't have a link, so check if discussion
-            # have one)
-            for object in reversed(breadcrumb):
-                link = object.spam_report_link()
-                if link:
-                    break
+            message = self.spam_report_message(breadcrumb)
 
-            on_new_potential_spam.send(self, title=title, link=link, text=text, reason=reason)
+            on_new_potential_spam.send(self, message=message, text=text, reason=reason)
 
             # We clean the listener here. Not sure if it's necessary either.
             signals.post_save.disconnect(report_after_save)
 
-        # For things like `spam_report_link` we often need the ID of the document so we
+        # For `spam_report_message` we often need the ID of the document so we
         # must report after saving to have the ID available.
         # By default the signal is weak so it is dropped at the end of this function and it's
         # never called. We disconnect the signal in `report_after_save` to avoid leaks.

udata/core/spam/tests/test_spam.py

@@ -0,0 +1,26 @@
+import logging
+import pytest
+from udata.tests import TestCase
+from udata.tests.helpers import assert_not_emit
+from udata.models import db
+
+from ..models import POTENTIAL_SPAM, SpamMixin
+
+
+log = logging.getLogger(__name__)
+
+class TestModel(SpamMixin, db.Document):
+    text = db.StringField(required=True)
+    _created = True
+
+    def texts_to_check_for_spam(self):
+        return [self.text]
+
+
+class SpamTest(TestCase):
+    @pytest.mark.options(SPAM_WORDS=['spam'], SPAM_ALLOWED_LANGS=['fr'])
+    def test_uppercase_lang_detect(self):
+        model = TestModel(text="DONNEES DE RECENSEMENT - MARCHES PUBLICS")
+        model.detect_spam()
+        self.assertNotEqual(model.spam.status, POTENTIAL_SPAM)
+

udata/harvest/backends/base.py

@@ -156,7 +156,7 @@ class BaseBackend(object):
             self.job.errors.append(error)
             self.job.status = 'failed'
             self.end()
-            return
+            return None
         except Exception as e:
             self.job.status = 'failed'
             error = HarvestError(message=safe_unicode(e))
@@ -164,7 +164,7 @@ class BaseBackend(object):
             self.end()
             msg = 'Initialization failed for "{0.name}" ({0.backend})'
             log.exception(msg.format(self.source))
-            return
+            return None
 
         if self.max_items:
             self.job.items = self.job.items[:self.max_items]

udata/harvest/models.py

@@ -94,7 +94,7 @@ class HarvestSource(db.Owned, db.Document):
                         populate_from='name', update=True)
     description = db.StringField()
     url = db.StringField(required=True)
-    backend = db.StringField()
+    backend = db.StringField(required=True)
     config = db.DictField()
     periodic_task = db.ReferenceField('PeriodicTask',
                                       reverse_delete_rule=db.NULLIFY)

udata/harvest/tasks.py

@@ -19,15 +19,18 @@ def harvest(self, ident):
     Backend = backends.get(current_app, source.backend)
     backend = Backend(source)
     items = backend.perform_initialization()
-    if items > 0:
+    if items is None:
+        pass
+    elif items == 0:
+        backend.finalize()
+    else:
         finalize = harvest_job_finalize.s(backend.job.id)
         items = [
             harvest_job_item.s(backend.job.id, item.remote_id)
             for item in backend.job.items
         ]
         chord(items)(finalize)
-    elif items == 0:
-        backend.finalize()
+    
 
 
 @task(ignore_result=False, route='low.harvest')

udata/harvest/tests/factories.py

@@ -22,6 +22,7 @@ class HarvestSourceFactory(ModelFactory):
     name = factory.Faker('name')
     url = factory.Faker('url')
     description = factory.Faker('text')
+    backend = 'factory'
 
 
 class HarvestJobFactory(ModelFactory):

udata/harvest/tests/test_models.py

@@ -12,7 +12,7 @@ log = logging.getLogger(__name__)
 @pytest.mark.usefixtures('clean_db')
 class HarvestSourceTest:
     def test_defaults(self):
-        source = HarvestSource.objects.create(name='Test', url=faker.url())
+        source = HarvestSource.objects.create(name='Test', url=faker.url(), backend='factory')
         assert source.name == 'Test'
         assert source.slug == 'test'
 

udata/notifications/mattermost.py

@@ -5,16 +5,11 @@ from flask import current_app
 
 @on_new_potential_spam.connect
 def notify_potential_spam(sender, **kwargs):
-    title = kwargs.get('title', 'no title')
-    link = kwargs.get('link')
+    message = kwargs.get('message')
     reason = kwargs.get('reason')
     text = kwargs.get('text')
 
-    message = ':warning: @all Spam potentiel sur '
-    if link:
-        message += f'[{title}]({link})'
-    else:
-        message += title
+    message = f':warning: @all {message}'
 
     if reason:
         message += f' ({reason})'

udata/settings.py

@@ -470,8 +470,7 @@ class Testing(object):
     WTF_CSRF_ENABLED = False
     AUTO_INDEX = False
     CELERY_TASK_ALWAYS_EAGER = True
-    # TODO: ideally, this should be set to True in order to reveal exceptions in delayed tasks
-    CELERY_TASK_EAGER_PROPAGATES = False
+    CELERY_TASK_EAGER_PROPAGATES = True
     TEST_WITH_PLUGINS = False
     PLUGINS = []
     TEST_WITH_THEME = False