udata 9.1.2.dev30355__py2.py3-none-any.whl → 9.1.2.dev30454__py2.py3-none-any.whl

udata/core/site/models.py

@@ -1,13 +1,12 @@
-from flask import g, current_app
+from flask import current_app, g
 from werkzeug.local import LocalProxy
 
-from udata.models import db, WithMetrics
-from udata.core.organization.models import Organization
 from udata.core.dataset.models import Dataset
+from udata.core.organization.models import Organization
 from udata.core.reuse.models import Reuse
+from udata.models import WithMetrics, db
 
-
-__all__ = ('Site', 'SiteSettings')
+__all__ = ("Site", "SiteSettings")
 
 
 DEFAULT_FEED_SIZE = 20
@@ -28,126 +27,158 @@ class Site(WithMetrics, db.Document):
     settings = db.EmbeddedDocumentField(SiteSettings, default=SiteSettings)
 
     __metrics_keys__ = [
-        'max_dataset_followers',
-        'max_dataset_reuses',
-        'max_reuse_datasets',
-        'max_reuse_followers',
-        'max_org_followers',
-        'max_org_reuses',
-        'max_org_datasets',
-        'datasets',
-        'discussions',
-        'followers',
-        'organizations',
-        'public-service',
-        'resources',
-        'reuses',
-        'users',
-        'harvesters'
+        "max_dataset_followers",
+        "max_dataset_reuses",
+        "max_reuse_datasets",
+        "max_reuse_followers",
+        "max_org_followers",
+        "max_org_reuses",
+        "max_org_datasets",
+        "datasets",
+        "discussions",
+        "followers",
+        "organizations",
+        "public-service",
+        "resources",
+        "reuses",
+        "users",
+        "harvesters",
     ]
 
     def __str__(self):
-        return self.title or ''
+        return self.title or ""
 
     def count_users(self):
         from udata.models import User
-        self.metrics['users'] = User.objects(confirmed_at__ne=None, deleted=None).count()
+
+        self.metrics["users"] = User.objects(confirmed_at__ne=None, deleted=None).count()
         self.save()
 
     def count_org(self):
         from udata.models import Organization
-        self.metrics['organizations'] = Organization.objects.visible().count()
+
+        self.metrics["organizations"] = Organization.objects.visible().count()
         self.save()
 
     def count_org_for_badge(self, badge_kind):
         from udata.models import Organization
+
         self.metrics[badge_kind] = Organization.objects(badges__kind=badge_kind).count()
         self.save()
 
     def count_datasets(self):
         from udata.models import Dataset
-        self.metrics['datasets'] = Dataset.objects.visible().count()
+
+        self.metrics["datasets"] = Dataset.objects.visible().count()
         self.save()
 
     def count_resources(self):
-        self.metrics['resources'] = next(Dataset.objects.visible().aggregate(
-            {'$project': {'resources': 1}},
-            {'$unwind': '$resources'},
-            {'$group': {'_id': 'result', 'count': {'$sum': 1}}}
-        ), {}).get('count', 0)
+        self.metrics["resources"] = next(
+            Dataset.objects.visible().aggregate(
+                {"$project": {"resources": 1}},
+                {"$unwind": "$resources"},
+                {"$group": {"_id": "result", "count": {"$sum": 1}}},
+            ),
+            {},
+        ).get("count", 0)
         self.save()
 
     def count_reuses(self):
-        self.metrics['reuses'] = Reuse.objects.visible().count()
+        self.metrics["reuses"] = Reuse.objects.visible().count()
         self.save()
 
     def count_followers(self):
         from udata.models import Follow
-        self.metrics['followers'] = Follow.objects(until=None).count()
+
+        self.metrics["followers"] = Follow.objects(until=None).count()
         self.save()
 
     def count_discussions(self):
         from udata.models import Discussion
-        self.metrics['discussions'] = Discussion.objects.count()
+
+        self.metrics["discussions"] = Discussion.objects.count()
         self.save()
 
     def count_harvesters(self):
         from udata.harvest.models import HarvestSource
-        self.metrics['harvesters'] = HarvestSource.objects().count()
+
+        self.metrics["harvesters"] = HarvestSource.objects().count()
         self.save()
 
     def count_max_dataset_followers(self):
-        dataset = (Dataset.objects(metrics__followers__gt=0).visible()
-                          .order_by('-metrics.followers').first())
-        self.metrics['max_dataset_followers'] = dataset.metrics['followers'] if dataset else 0
+        dataset = (
+            Dataset.objects(metrics__followers__gt=0)
+            .visible()
+            .order_by("-metrics.followers")
+            .first()
+        )
+        self.metrics["max_dataset_followers"] = dataset.metrics["followers"] if dataset else 0
         self.save()
 
     def count_max_dataset_reuses(self):
-        dataset = (Dataset.objects(metrics__reuses__gt=0).visible()
-                   .order_by('-metrics.reuses').first())
-        self.metrics['max_dataset_reuses'] = dataset.metrics['reuses'] if dataset else 0
+        dataset = (
+            Dataset.objects(metrics__reuses__gt=0).visible().order_by("-metrics.reuses").first()
+        )
+        self.metrics["max_dataset_reuses"] = dataset.metrics["reuses"] if dataset else 0
         self.save()
 
     def count_max_reuse_datasets(self):
-        reuse = (Reuse.objects(metrics__datasets__gt=0).visible()
-                 .order_by('-metrics.datasets').first())
-        self.metrics['max_reuse_datasets'] = reuse.metrics['datasets'] if reuse else 0
+        reuse = (
+            Reuse.objects(metrics__datasets__gt=0).visible().order_by("-metrics.datasets").first()
+        )
+        self.metrics["max_reuse_datasets"] = reuse.metrics["datasets"] if reuse else 0
         self.save()
 
     def count_max_reuse_followers(self):
-        reuse = (Reuse.objects(metrics__followers__gt=0).visible()
-                 .order_by('-metrics.followers').first())
-        self.metrics['max_reuse_followers'] = reuse.metrics['followers'] if reuse else 0
+        reuse = (
+            Reuse.objects(metrics__followers__gt=0).visible().order_by("-metrics.followers").first()
+        )
+        self.metrics["max_reuse_followers"] = reuse.metrics["followers"] if reuse else 0
         self.save()
 
     def count_max_org_followers(self):
-        org = (Organization.objects(metrics__followers__gt=0).visible()
-               .order_by('-metrics.followers').first())
-        self.metrics['max_org_followers'] = org.metrics['followers'] if org else 0
+        org = (
+            Organization.objects(metrics__followers__gt=0)
+            .visible()
+            .order_by("-metrics.followers")
+            .first()
+        )
+        self.metrics["max_org_followers"] = org.metrics["followers"] if org else 0
         self.save()
 
     def count_max_org_reuses(self):
-        org = (Organization.objects(metrics__reuses__gt=0).visible()
-               .order_by('-metrics.reuses').first())
-        self.metrics['max_org_reuses'] = org.metrics['reuses'] if org else 0
+        org = (
+            Organization.objects(metrics__reuses__gt=0)
+            .visible()
+            .order_by("-metrics.reuses")
+            .first()
+        )
+        self.metrics["max_org_reuses"] = org.metrics["reuses"] if org else 0
         self.save()
 
     def count_max_org_datasets(self):
-        org = (Organization.objects(metrics__datasets__gt=0).visible()
-               .order_by('-metrics.datasets').first())
-        self.metrics['max_org_datasets'] = org.metrics['datasets'] if org else 0
+        org = (
+            Organization.objects(metrics__datasets__gt=0)
+            .visible()
+            .order_by("-metrics.datasets")
+            .first()
+        )
+        self.metrics["max_org_datasets"] = org.metrics["datasets"] if org else 0
         self.save()
 
 
 def get_current_site():
-    if getattr(g, 'site', None) is None:
-        site_id = current_app.config['SITE_ID']
-        site_title = current_app.config.get('SITE_TITLE')
-        site_keywords = current_app.config.get('SITE_KEYWORDS', [])
-        g.site, _ = Site.objects.get_or_create(id=site_id, defaults={
-            'title': site_title,
-            'keywords': site_keywords,
-        })
+    if getattr(g, "site", None) is None:
+        site_id = current_app.config["SITE_ID"]
+        site_title = current_app.config.get("SITE_TITLE")
+        site_keywords = current_app.config.get("SITE_KEYWORDS", [])
+        g.site, _ = Site.objects.get_or_create(
+            id=site_id,
+            defaults={
+                "title": site_title,
+                "keywords": site_keywords,
+            },
+        )
         if g.site.title != site_title:
             Site.objects(id=site_id).modify(set__title=site_title)
         if g.site.keywords != site_keywords:

udata/core/site/rdf.py

@@ -1,36 +1,36 @@
-'''
+"""
 This module centralize site helpers for RDF/DCAT serialization and parsing
-'''
-from flask import url_for, current_app
-from rdflib import Graph, URIRef, Literal, BNode
-from rdflib.namespace import RDF, FOAF
+"""
+
+from flask import current_app, url_for
+from rdflib import BNode, Graph, Literal, URIRef
+from rdflib.namespace import FOAF, RDF
 
 from udata.core.dataservices.rdf import dataservice_to_rdf
 from udata.core.dataset.rdf import dataset_to_rdf
 from udata.core.organization.rdf import organization_to_rdf
 from udata.core.user.rdf import user_to_rdf
 from udata.rdf import DCAT, DCT, namespace_manager, paginate_catalog
-from udata.utils import Paginable
 from udata.uris import endpoint_for
+from udata.utils import Paginable
 
 
-def build_catalog(site, datasets, dataservices = [], format=None):
-    '''Build the DCAT catalog for this site'''
-    site_url = endpoint_for('site.home_redirect', 'api.site', _external=True)
-    catalog_url = url_for('api.site_rdf_catalog', _external=True)
+def build_catalog(site, datasets, dataservices=[], format=None):
+    """Build the DCAT catalog for this site"""
+    site_url = endpoint_for("site.home_redirect", "api.site", _external=True)
+    catalog_url = url_for("api.site_rdf_catalog", _external=True)
     graph = Graph(namespace_manager=namespace_manager)
     catalog = graph.resource(URIRef(catalog_url))
 
     catalog.set(RDF.type, DCAT.Catalog)
     catalog.set(DCT.title, Literal(site.title))
     catalog.set(DCT.description, Literal(f"{site.title}"))
-    catalog.set(DCT.language,
-                Literal(current_app.config['DEFAULT_LANGUAGE']))
+    catalog.set(DCT.language, Literal(current_app.config["DEFAULT_LANGUAGE"]))
     catalog.set(FOAF.homepage, URIRef(site_url))
 
     publisher = graph.resource(BNode())
     publisher.set(RDF.type, FOAF.Organization)
-    publisher.set(FOAF.name, Literal(current_app.config['SITE_AUTHOR']))
+    publisher.set(FOAF.name, Literal(current_app.config["SITE_AUTHOR"]))
     catalog.set(DCT.publisher, publisher)
 
     for dataset in datasets:
@@ -46,6 +46,6 @@ def build_catalog(site, datasets, dataservices = [], format=None):
         catalog.add(DCAT.DataService, rdf_dataservice)
 
     if isinstance(datasets, Paginable):
-        paginate_catalog(catalog, graph, datasets, format, 'api.site_rdf_catalog_format')
+        paginate_catalog(catalog, graph, datasets, format, "api.site_rdf_catalog_format")
 
     return catalog

udata/core/spam/api.py

@@ -1,10 +1,10 @@
 from mongoengine import Q
 
-from udata.api import api, API
+from udata.api import API, api
 from udata.auth import admin_permission
 from udata.core.discussions.models import Discussion
-from udata.core.spam.fields import potential_spam_fields
 from udata.core.spam.constants import POTENTIAL_SPAM
+from udata.core.spam.fields import potential_spam_fields
 from udata.utils import id_or_404
 
 
@@ -12,6 +12,7 @@ class SpamAPIMixin(API):
     """
     Base Spam Model API.
     """
+
     model = None
 
     def get_model(self, id):
@@ -37,21 +38,27 @@ class SpamAPIMixin(API):
         return {}, 200
 
 
-ns = api.namespace('spam', 'Spam related operations')
+ns = api.namespace("spam", "Spam related operations")
 
 
-@ns.route('/', endpoint='spam')
+@ns.route("/", endpoint="spam")
 class SpamAPI(API):
     """
     Base class for a discussion thread.
     """
-    @api.doc('get_potential_spams')
+
+    @api.doc("get_potential_spams")
     @api.secure(admin_permission)
     @api.marshal_with(potential_spam_fields)
     def get(self):
         """Get all potential spam objects"""
-        discussions = Discussion.objects(Q(spam__status=POTENTIAL_SPAM) | Q(discussion__spam__status=POTENTIAL_SPAM))
+        discussions = Discussion.objects(
+            Q(spam__status=POTENTIAL_SPAM) | Q(discussion__spam__status=POTENTIAL_SPAM)
+        )
 
-        return [{
-            'message': discussion.spam_report_message([discussion]),
-        } for discussion in discussions]
+        return [
+            {
+                "message": discussion.spam_report_message([discussion]),
+            }
+            for discussion in discussions
+        ]

udata/core/spam/constants.py

@@ -1,5 +1,5 @@
-NOT_CHECKED = 'not_checked'
-POTENTIAL_SPAM = 'potential_spam'
-NO_SPAM = 'no_spam'
+NOT_CHECKED = "not_checked"
+POTENTIAL_SPAM = "potential_spam"
+NO_SPAM = "no_spam"
 
-SPAM_STATUS_CHOICES = [NOT_CHECKED, POTENTIAL_SPAM, NO_SPAM]
+SPAM_STATUS_CHOICES = [NOT_CHECKED, POTENTIAL_SPAM, NO_SPAM]

udata/core/spam/fields.py

@@ -1,11 +1,17 @@
 from udata.api import api, fields
-from .constants import SPAM_STATUS_CHOICES
 
-spam_fields = api.model('Spam', {
-    'status': fields.String(description='Status', enum=SPAM_STATUS_CHOICES, readonly=True),
-})
+from .constants import SPAM_STATUS_CHOICES
 
-potential_spam_fields = api.model('PotentialSpam', {
-    'message': fields.String(readonly=True),
-})
+spam_fields = api.model(
+    "Spam",
+    {
+        "status": fields.String(description="Status", enum=SPAM_STATUS_CHOICES, readonly=True),
+    },
+)
 
+potential_spam_fields = api.model(
+    "PotentialSpam",
+    {
+        "message": fields.String(readonly=True),
+    },
+)

udata/core/spam/models.py

@@ -1,11 +1,11 @@
 from flask import current_app
 from langdetect import detect
+from mongoengine import signals
 
 from udata.mongo import db
-from .signals import on_new_potential_spam
-from mongoengine import signals
 
-from .constants import NO_SPAM, POTENTIAL_SPAM, SPAM_STATUS_CHOICES, NOT_CHECKED
+from .constants import NO_SPAM, NOT_CHECKED, POTENTIAL_SPAM, SPAM_STATUS_CHOICES
+from .signals import on_new_potential_spam
 
 
 class SpamInfo(db.EmbeddedDocument):
@@ -28,16 +28,15 @@ class SpamMixin(object):
 
     @staticmethod
     def spam_words():
-        return current_app.config.get('SPAM_WORDS', [])
+        return current_app.config.get("SPAM_WORDS", [])
 
     @staticmethod
     def allowed_langs():
-        return current_app.config.get('SPAM_ALLOWED_LANGS', [])
-    
+        return current_app.config.get("SPAM_ALLOWED_LANGS", [])
+
     @staticmethod
     def minimum_string_length_for_lang_check():
-        return current_app.config.get('SPAM_MINIMUM_STRING_LENGTH_FOR_LANG_CHECK', 30)
-
+        return current_app.config.get("SPAM_MINIMUM_STRING_LENGTH_FOR_LANG_CHECK", 30)
 
     def clean(self):
         super().clean()
@@ -91,15 +90,22 @@ class SpamMixin(object):
             for word in SpamMixin.spam_words():
                 if word in text.lower():
                     self.spam.status = POTENTIAL_SPAM
-                    self._report(text=text, breadcrumb=breadcrumb, reason=f"contains spam words \"{word}\"")
+                    self._report(
+                        text=text, breadcrumb=breadcrumb, reason=f'contains spam words "{word}"'
+                    )
                     return
 
             # Language detection is not working well with texts of a few words.
-            if SpamMixin.allowed_langs() and len(text) > SpamMixin.minimum_string_length_for_lang_check():
+            if (
+                SpamMixin.allowed_langs()
+                and len(text) > SpamMixin.minimum_string_length_for_lang_check()
+            ):
                 lang = detect(text.lower())
                 if lang not in SpamMixin.allowed_langs():
                     self.spam.status = POTENTIAL_SPAM
-                    self._report(text=text, breadcrumb=breadcrumb, reason=f"not allowed language \"{lang}\"")
+                    self._report(
+                        text=text, breadcrumb=breadcrumb, reason=f'not allowed language "{lang}"'
+                    )
                     return
 
         for embed in self.embeds_to_check_for_spam():
@@ -130,19 +136,20 @@ class SpamMixin(object):
 
         for name, args in callbacks.items():
             callback = getattr(base_model, name)
-            callback(*args['args'], **args['kwargs'])
+            callback(*args["args"], **args["kwargs"])
 
     def is_spam(self):
         return self.spam and self.spam.status == POTENTIAL_SPAM
 
     def texts_to_check_for_spam(self):
         raise NotImplementedError(
-            "Please implement the `texts_to_check_for_spam` method. Should return a list of strings to check.")
+            "Please implement the `texts_to_check_for_spam` method. Should return a list of strings to check."
+        )
 
     def embeds_to_check_for_spam(self):
         return []
 
-    def spam_is_whitelisted(self) -> bool :
+    def spam_is_whitelisted(self) -> bool:
         return False
 
     def spam_report_message(self):
@@ -177,7 +184,7 @@ def spam_protected(get_model_to_check=None):
     It will save the class method called with its arguments inside the `SpamInfo` object to be
     called later if needed (in case of false positive).
     The decorator accept an argument, a function to get the model to check when we are doing an operation
-    on an embed document. The class method should always take a `self` as a first argument which is the base 
+    on an embed document. The class method should always take a `self` as a first argument which is the base
     model to allow saving the callbacks back into Mongo (we cannot .save() an embed document).
     """
 
@@ -191,13 +198,13 @@ def spam_protected(get_model_to_check=None):
 
             if not isinstance(model_to_check, SpamMixin):
                 raise ValueError(
-                    "@spam_protected should be called within a SpamMixin. " + type(model_to_check).__name__ + " given.")
+                    "@spam_protected should be called within a SpamMixin. "
+                    + type(model_to_check).__name__
+                    + " given."
+                )
 
             if model_to_check.is_spam():
-                model_to_check.spam.callbacks[f.__name__] = {
-                    'args': args[1:],
-                    'kwargs': kwargs
-                }
+                model_to_check.spam.callbacks[f.__name__] = {"args": args[1:], "kwargs": kwargs}
                 base_model.save_without_spam_detection()
             else:
                 f(*args, **kwargs)

udata/core/spam/signals.py

@@ -3,4 +3,4 @@ from blinker import Namespace
 namespace = Namespace()
 
 #: Triggered when a spam is detected
-on_new_potential_spam = namespace.signal('on-new-potential-spam')
+on_new_potential_spam = namespace.signal("on-new-potential-spam")

udata/core/spam/tests/test_spam.py

@@ -1,14 +1,16 @@
 import logging
+
 import pytest
-from udata.tests import TestCase
+
 from udata.mongo import db
+from udata.tests import TestCase
 
-from ..models import SpamMixin
 from ..constants import POTENTIAL_SPAM
-
+from ..models import SpamMixin
 
 log = logging.getLogger(__name__)
 
+
 class TestModel(SpamMixin, db.Document):
     text = db.StringField(required=True)
     _created = True
@@ -18,9 +20,8 @@ class TestModel(SpamMixin, db.Document):
 
 
 class SpamTest(TestCase):
-    @pytest.mark.options(SPAM_WORDS=['spam'], SPAM_ALLOWED_LANGS=['fr'])
+    @pytest.mark.options(SPAM_WORDS=["spam"], SPAM_ALLOWED_LANGS=["fr"])
     def test_uppercase_lang_detect(self):
         model = TestModel(text="DONNEES DE RECENSEMENT - MARCHES PUBLICS")
         model.detect_spam()
         self.assertNotEqual(model.spam.status, POTENTIAL_SPAM)
-