udata 9.1.2.dev30355__py2.py3-none-any.whl → 9.1.2.dev30454__py2.py3-none-any.whl

udata/linkchecker/checker.py

@@ -2,27 +2,28 @@ from urllib.parse import urlparse
 
 from flask import current_app
 
-from .backends import get as get_linkchecker, NoCheckLinkchecker
+from .backends import NoCheckLinkchecker
+from .backends import get as get_linkchecker
 
 
 def _get_check_keys(the_dict, resource, previous_status):
-    check_keys = {k: v for k, v in the_dict.items()
-                  if k.startswith('check:')}
-    check_keys['check:count-availability'] = _compute_count_availability(
-            resource, check_keys.get('check:available'), previous_status)
+    check_keys = {k: v for k, v in the_dict.items() if k.startswith("check:")}
+    check_keys["check:count-availability"] = _compute_count_availability(
+        resource, check_keys.get("check:available"), previous_status
+    )
     return check_keys
 
 
 def _compute_count_availability(resource, status, previous_status):
-    '''Compute the `check:count-availability` extra value'''
-    count_availability = resource.extras.get('check:count-availability', 1)
+    """Compute the `check:count-availability` extra value"""
+    count_availability = resource.extras.get("check:count-availability", 1)
     return count_availability + 1 if status == previous_status else 1
 
 
 def is_ignored(resource):
-    '''Check if the resource's URL is to be ignored'''
-    ignored_domains = current_app.config['LINKCHECKING_IGNORE_DOMAINS']
-    ignored_patterns = current_app.config['LINKCHECKING_IGNORE_PATTERNS']
+    """Check if the resource's URL is to be ignored"""
+    ignored_domains = current_app.config["LINKCHECKING_IGNORE_DOMAINS"]
+    ignored_patterns = current_app.config["LINKCHECKING_IGNORE_PATTERNS"]
     url = resource.url
     if not url:
         return True
@@ -33,12 +34,12 @@ def is_ignored(resource):
 
 
 def dummy_check_response():
-    '''Trigger a dummy check'''
+    """Trigger a dummy check"""
     return NoCheckLinkchecker().check(None)
 
 
 def check_resource(resource):
-    '''
+    """
     Check a resource availability against a linkchecker backend
 
     The linkchecker used can be configured on a resource basis by setting
@@ -51,24 +52,24 @@ def check_resource(resource):
     -------
     dict or (dict, int)
         Check results dict and status code (if error).
-    '''
-    linkchecker_type = resource.extras.get('check:checker')
+    """
+    linkchecker_type = resource.extras.get("check:checker")
     LinkChecker = get_linkchecker(linkchecker_type)
     if not LinkChecker:
-        return {'error': 'No linkchecker configured.'}, 503
+        return {"error": "No linkchecker configured."}, 503
     if is_ignored(resource):
         return dummy_check_response()
     result = LinkChecker().check(resource)
     if not result:
-        return {'error': 'No response from linkchecker'}, 503
-    elif result.get('check:error'):
-        return {'error': result['check:error']}, 500
-    elif not result.get('check:status'):
-        return {'error': 'No status in response from linkchecker'}, 503
+        return {"error": "No response from linkchecker"}, 503
+    elif result.get("check:error"):
+        return {"error": result["check:error"]}, 500
+    elif not result.get("check:status"):
+        return {"error": "No status in response from linkchecker"}, 503
     # store the check result in the resource's extras
     # XXX maybe this logic should be in the `Resource` model?
-    previous_status = resource.extras.get('check:available')
+    previous_status = resource.extras.get("check:available")
     check_keys = _get_check_keys(result, resource, previous_status)
     resource.extras.update(check_keys)
-    resource.save(signal_kwargs={'ignores': ['post_save']})  # Prevent signal triggering on dataset
+    resource.save(signal_kwargs={"ignores": ["post_save"]})  # Prevent signal triggering on dataset
     return result

udata/linkchecker/commands.py

@@ -5,19 +5,17 @@ import click
 from udata.commands import cli
 from udata.linkchecker.tasks import check_resources
 
-
 log = logging.getLogger(__name__)
 
 
-@cli.group('linkchecker')
+@cli.group("linkchecker")
 def grp():
-    '''Link checking operations'''
+    """Link checking operations"""
     pass
 
 
 @grp.command()
-@click.option('-n', '--number', type=int, default=5000,
-              help='Number of URLs to check')
+@click.option("-n", "--number", type=int, default=5000, help="Number of URLs to check")
 def check(number):
-    '''Check <number> of URLs that have not been (recently) checked'''
+    """Check <number> of URLs that have not been (recently) checked"""
     check_resources(number)

udata/linkchecker/models.py

@@ -1,9 +1,9 @@
-from udata.mongo import db
 from udata.core.dataset.models import ResourceMixin
+from udata.mongo import db
 
 # Register harvest extras
-ResourceMixin.extras.register('check:available', db.BooleanField)
-ResourceMixin.extras.register('check:count-availability', db.IntField)
-ResourceMixin.extras.register('check:status', db.IntField)
-ResourceMixin.extras.register('check:url', db.StringField)
-ResourceMixin.extras.register('check:date', db.DateTimeField)
+ResourceMixin.extras.register("check:available", db.BooleanField)
+ResourceMixin.extras.register("check:count-availability", db.IntField)
+ResourceMixin.extras.register("check:status", db.IntField)
+ResourceMixin.extras.register("check:url", db.StringField)
+ResourceMixin.extras.register("check:date", db.DateTimeField)

udata/linkchecker/tasks.py

@@ -12,46 +12,44 @@ from .checker import check_resource
 log = logging.getLogger(__name__)
 
 
-@job('check_resources')
+@job("check_resources")
 def check_resources(self, number):
-    '''Check <number> of URLs that have not been (recently) checked'''
-    if not current_app.config.get('LINKCHECKING_ENABLED'):
-        log.error('Link checking is disabled.')
+    """Check <number> of URLs that have not been (recently) checked"""
+    if not current_app.config.get("LINKCHECKING_ENABLED"):
+        log.error("Link checking is disabled.")
         return
 
     base_pipeline = [
-        {'$match': {'resources': {'$gt': []}}},
-        {'$project': {'resources._id': True,
-                      'resources.extras.check:date': True}},
-        {'$unwind': '$resources'},
+        {"$match": {"resources": {"$gt": []}}},
+        {"$project": {"resources._id": True, "resources.extras.check:date": True}},
+        {"$unwind": "$resources"},
     ]
     # unchecked resources
     pipeline = base_pipeline + [
-        {'$match': {'resources.extras.check:date': {'$eq': None}}},
-        {'$limit': number}
+        {"$match": {"resources.extras.check:date": {"$eq": None}}},
+        {"$limit": number},
     ]
     resources = list(Dataset.objects.aggregate(*pipeline))
     # not recently checked resources
     slots_left = number - len(resources)
     if slots_left:
         pipeline = base_pipeline + [
-            {'$match': {'resources.extras.check:date': {'$ne': None}}},
-            {'$sort': {'resources.extras.check:date': 1}},
-            {'$limit': slots_left}
+            {"$match": {"resources.extras.check:date": {"$ne": None}}},
+            {"$sort": {"resources.extras.check:date": 1}},
+            {"$limit": slots_left},
         ]
         resources += list(Dataset.objects.aggregate(*pipeline))
 
     nb_resources = len(resources)
-    log.info('Checking %s resources...', nb_resources)
+    log.info("Checking %s resources...", nb_resources)
     for idx, dataset_resource in enumerate(resources):
-        dataset_obj = Dataset.objects.get(id=dataset_resource['_id'])
-        resource_id = dataset_resource['resources']['_id']
+        dataset_obj = Dataset.objects.get(id=dataset_resource["_id"])
+        resource_id = dataset_resource["resources"]["_id"]
         rid = uuid.UUID(resource_id)
-        resource_obj = get_by(dataset_obj.resources, 'id', rid)
-        log.info('Checking resource %s (%s/%s)',
-                 resource_id, idx + 1, nb_resources)
+        resource_obj = get_by(dataset_obj.resources, "id", rid)
+        log.info("Checking resource %s (%s/%s)", resource_id, idx + 1, nb_resources)
         if resource_obj.need_check():
             check_resource(resource_obj)
         else:
             log.info("--> Skipping this resource, cache is fresh enough.")
-    log.info('Done.')
+    log.info("Done.")

udata/mail.py

@@ -1,25 +1,23 @@
 import logging
 from contextlib import contextmanager
+from smtplib import SMTPException
 
 from blinker import signal
-
 from flask import current_app, render_template
 from flask_mail import Mail, Message
 
-from smtplib import SMTPException
-
 from udata import i18n
 
-
 log = logging.getLogger(__name__)
 
 mail = Mail()
 
-mail_sent = signal('mail-sent')
+mail_sent = signal("mail-sent")
 
 
 class FakeMailer(object):
-    '''Display sent mail in logging output'''
+    """Display sent mail in logging output"""
+
     def send(self, msg):
         log.debug(msg.body)
         log.debug(msg.html)
@@ -37,37 +35,39 @@ def init_app(app):
 
 
 def send(subject, recipients, template_base, **kwargs):
-    '''
+    """
     Send a given email to multiple recipients.
 
     User prefered language is taken in account.
     To translate the subject in the right language, you should ugettext_lazy
-    '''
-    sender = kwargs.pop('sender', None)
+    """
+    sender = kwargs.pop("sender", None)
     if not isinstance(recipients, (list, tuple)):
         recipients = [recipients]
 
-    tpl_path = f'mail/{template_base}'
+    tpl_path = f"mail/{template_base}"
 
-    debug = current_app.config.get('DEBUG', False)
-    send_mail = current_app.config.get('SEND_MAIL', not debug)
+    debug = current_app.config.get("DEBUG", False)
+    send_mail = current_app.config.get("SEND_MAIL", not debug)
     connection = send_mail and mail.connect or dummyconnection
 
     with connection() as conn:
         for recipient in recipients:
             lang = i18n._default_lang(recipient)
             with i18n.language(lang):
-                log.debug(
-                    'Sending mail "%s" to recipient "%s"', subject, recipient)
-                msg = Message(subject, sender=sender,
-                              recipients=[recipient.email])
+                log.debug('Sending mail "%s" to recipient "%s"', subject, recipient)
+                msg = Message(subject, sender=sender, recipients=[recipient.email])
                 msg.body = render_template(
-                    f'{tpl_path}.txt', subject=subject,
-                    sender=sender, recipient=recipient, **kwargs)
+                    f"{tpl_path}.txt", subject=subject, sender=sender, recipient=recipient, **kwargs
+                )
                 msg.html = render_template(
-                    f'{tpl_path}.html', subject=subject,
-                    sender=sender, recipient=recipient, **kwargs)
+                    f"{tpl_path}.html",
+                    subject=subject,
+                    sender=sender,
+                    recipient=recipient,
+                    **kwargs,
+                )
                 try:
                     conn.send(msg)
                 except SMTPException as e:
-                    log.error(f'Error sending mail {e}')
+                    log.error(f"Error sending mail {e}")

udata/migrations/2020-07-24-remove-s-from-scope-oauth.py

@@ -1,8 +1,9 @@
-'''
+"""
 The purpose here is to change the scopes attribute of
 the OAuth2Client to scope without the s, then
 to turn the list into a string.
-'''
+"""
+
 import logging
 
 from mongoengine.connection import get_db
@@ -11,15 +12,15 @@ log = logging.getLogger(__name__)
 
 
 def migrate(db):
-    log.info('Processing OAuth2Client objects.')
+    log.info("Processing OAuth2Client objects.")
 
     db = get_db()
     oauth_clients = db.oauth2_client
-    oauth_clients.update_many({}, {'$rename': {'scopes': 'scope'}})
+    oauth_clients.update_many({}, {"$rename": {"scopes": "scope"}})
     for client in oauth_clients.find():
-        if type(client['scope']) == list:
-            scope_str = ' '.join(client['scope'])
-            client['scope'] = scope_str
+        if type(client["scope"]) == list:
+            scope_str = " ".join(client["scope"])
+            client["scope"] = scope_str
             oauth_clients.save(client)
 
-    log.info('Completed.')
+    log.info("Completed.")

udata/migrations/2020-08-24-add-fs-filename.py

@@ -1,17 +1,18 @@
-'''
+"""
 The purpose here is to fill every resource with a fs_filename string field.
-'''
+"""
+
 import logging
 from urllib.parse import urlparse
 
 from udata.core import storages
-from udata.models import Dataset, CommunityResource
+from udata.models import CommunityResource, Dataset
 
 log = logging.getLogger(__name__)
 
 
 def migrate(db):
-    log.info('Processing resources.')
+    log.info("Processing resources.")
 
     datasets = Dataset.objects().no_cache().timeout(False)
     for dataset in datasets:
@@ -19,7 +20,7 @@ def migrate(db):
         for resource in dataset.resources:
             if resource.url.startswith(storages.resources.base_url):
                 parsed = urlparse(resource.url)
-                fs_name = parsed.path.replace('/resources/', '')
+                fs_name = parsed.path.replace("/resources/", "")
                 resource.fs_filename = fs_name
                 save_res = True
             elif resource.fs_filename is not None:
@@ -32,14 +33,14 @@ def migrate(db):
                 log.warning(e)
                 pass
 
-    log.info('Processing community resources.')
+    log.info("Processing community resources.")
 
     community_resources = CommunityResource.objects().no_cache().timeout(False)
     for community_resource in community_resources:
         save_res = False
         if community_resource.url.startswith(storages.resources.base_url):
             parsed = urlparse(community_resource.url)
-            fs_name = parsed.path.replace('/resources/', '')
+            fs_name = parsed.path.replace("/resources/", "")
             community_resource.fs_filename = fs_name
             save_res = True
         elif community_resource.fs_filename is not None:
@@ -52,4 +53,4 @@ def migrate(db):
                 log.warning(e)
                 pass
 
-    log.info('Completed.')
+    log.info("Completed.")

udata/migrations/2020-09-28-update-reuses-datasets-metrics.py

@@ -1,7 +1,8 @@
-'''
+"""
 The purpose here is to update every reuse's datasets metrics
 after the PR https://github.com/opendatateam/udata/pull/2531
-'''
+"""
+
 import logging
 
 from udata.models import Reuse
@@ -10,10 +11,10 @@ log = logging.getLogger(__name__)
 
 
 def migrate(db):
-    log.info('Processing reuses.')
+    log.info("Processing reuses.")
 
     reuses = Reuse.objects().no_cache().timeout(False)
     for reuse in reuses:
         reuse.count_datasets()
 
-    log.info('Completed.')
+    log.info("Completed.")

udata/migrations/2020-10-16-migrate-ods-resources.py

@@ -1,27 +1,26 @@
-'''
+"""
 Migrate ODS harvested resources URLS from
 use_labels_for_header=true to use_labels_for_header=false
 cf https://github.com/opendatateam/udata-ods/pull/169
-'''
+"""
+
 import logging
 
 from mongoengine.errors import ValidationError
+
 from udata.models import Dataset
 
 log = logging.getLogger(__name__)
 
-MIX_MATCH = [
-    'use_labels_for_header=true',
-    'use_labels_for_header=false'
-]
+MIX_MATCH = ["use_labels_for_header=true", "use_labels_for_header=false"]
 
 
 def migrate(db):
-    log.info('Migrating ODS harvest datasets...')
+    log.info("Migrating ODS harvest datasets...")
 
     count = 0
     # datasets from ODS
-    datasets = Dataset.objects.filter(**{'extras__ods:url__ne': None})
+    datasets = Dataset.objects.filter(**{"extras__ods:url__ne": None})
     for d in datasets:
         touched = False
         for r in d.resources:
@@ -33,6 +32,6 @@ def migrate(db):
             try:
                 d.save()
             except ValidationError as e:
-                log.warning(f'Error while saving dataset {d.id}: {str(e)}')
+                log.warning(f"Error while saving dataset {d.id}: {str(e)}")
 
-    log.info(f'Completed, {count} resources migrated.')
+    log.info(f"Completed, {count} resources migrated.")

udata/migrations/2021-04-08-update-schema-with-new-structure.py

@@ -1,7 +1,8 @@
-'''
+"""
 The purpose here is to update every resource's metadata 'schema'
 with a new format (string to object)
-'''
+"""
+
 import logging
 
 from udata.models import Dataset
@@ -10,17 +11,17 @@ log = logging.getLogger(__name__)
 
 
 def migrate(db):
-    log.info('Processing resources.')
+    log.info("Processing resources.")
 
     datasets = Dataset.objects().no_cache().timeout(False)
     for dataset in datasets:
         save_res = False
         for resource in dataset.resources:
-            if hasattr(resource, 'schema'):
+            if hasattr(resource, "schema"):
                 schema = resource.schema
-                resource.schema = {'name': None}
+                resource.schema = {"name": None}
                 if schema is not None and isinstance(schema, str):
-                    resource.schema = {'name': schema}
+                    resource.schema = {"name": schema}
                 save_res = True
         if save_res:
             try:
@@ -29,4 +30,4 @@ def migrate(db):
                 log.warning(e)
                 pass
 
-    log.info('Completed.')
+    log.info("Completed.")

udata/migrations/2021-05-27-fix-default-schema-name.py

@@ -1,6 +1,7 @@
-'''
+"""
 The purpose here is to update every resource's metadata 'schema' name.
-'''
+"""
+
 import logging
 
 from udata.models import Dataset
@@ -9,15 +10,15 @@ log = logging.getLogger(__name__)
 
 
 def migrate(db):
-    log.info('Processing resources.')
+    log.info("Processing resources.")
 
     datasets = Dataset.objects().no_cache().timeout(False)
     for dataset in datasets:
         save_res = False
         for resource in dataset.resources:
-            if hasattr(resource, 'schema'):
+            if hasattr(resource, "schema"):
                 for key, value in resource.schema.items():
-                    if key == 'name' and value is None:
+                    if key == "name" and value is None:
                         resource.schema = {}
                         save_res = True
         if save_res:
@@ -27,4 +28,4 @@ def migrate(db):
                 log.warning(e)
                 pass
 
-    log.info('Completed.')
+    log.info("Completed.")

udata/migrations/2021-07-05-remove-unused-badges.py

@@ -1,27 +1,29 @@
-'''
+"""
 The purpose here is to update every resource's metadata 'schema' name.
-'''
+"""
+
 import logging
-from udata.core.reuse.models import Reuse
 
+from udata.core.reuse.models import Reuse
 from udata.models import Dataset, Reuse
 
 log = logging.getLogger(__name__)
 
 UNUSED_BADGES = [
-    'dataconnexions-5-candidate',
-    'dataconnexions-5-laureate',
-    'dataconnexions-6-candidate',
-    'dataconnexions-6-laureate',
-    'covid-19',
-    'c3',
-    'nec',
-    'openfield16',
-    'bal'
+    "dataconnexions-5-candidate",
+    "dataconnexions-5-laureate",
+    "dataconnexions-6-candidate",
+    "dataconnexions-6-laureate",
+    "covid-19",
+    "c3",
+    "nec",
+    "openfield16",
+    "bal",
 ]
 
+
 def migrate(db):
-    log.info('Processing datasets.')
+    log.info("Processing datasets.")
 
     datasets = Dataset.objects().no_cache().timeout(False)
     for dataset in datasets:
@@ -29,7 +31,7 @@ def migrate(db):
             if dataset.get_badge(badge):
                 dataset.remove_badge(badge)
 
-    log.info('Processing reuses.')
+    log.info("Processing reuses.")
 
     reuses = Reuse.objects().no_cache().timeout(False)
     for reuse in reuses:
@@ -37,4 +39,4 @@ def migrate(db):
             if reuse.get_badge(badge):
                 reuse.remove_badge(badge)
 
-    log.info('Completed.')
+    log.info("Completed.")

udata/migrations/2021-07-07-update-schema-for-community-resources.py

@@ -1,7 +1,8 @@
-'''
+"""
 The purpose here is to update every community resource's metadata 'schema'
 with a new format (string to object)
-'''
+"""
+
 import logging
 
 from udata.models import CommunityResource
@@ -10,16 +11,16 @@ log = logging.getLogger(__name__)
 
 
 def migrate(db):
-    log.info('Processing community resources.')
+    log.info("Processing community resources.")
 
     community_resources = CommunityResource.objects().no_cache().timeout(False)
     for resource in community_resources:
         save_res = False
-        if hasattr(resource, 'schema'):
+        if hasattr(resource, "schema"):
             schema = resource.schema
             resource.schema = {}
             if schema is not None and isinstance(schema, str):
-                resource.schema = {'name': schema}
+                resource.schema = {"name": schema}
             save_res = True
         if save_res:
             try:
@@ -28,4 +29,4 @@ def migrate(db):
                 log.warning(e)
                 pass
 
-    log.info('Completed.')
+    log.info("Completed.")

udata/migrations/2021-08-17-follow-integrity.py

@@ -1,7 +1,8 @@
-'''
+"""
 Remove Follow integrity problems
 ⚠️ long migration
-'''
+"""
+
 import logging
 
 import mongoengine
@@ -12,7 +13,7 @@ log = logging.getLogger(__name__)
 
 
 def migrate(db):
-    log.info('Processing Follow references.')
+    log.info("Processing Follow references.")
 
     count = 0
     follows = Follow.objects(following__ne=None).no_cache().all()
@@ -23,4 +24,4 @@ def migrate(db):
             count += 1
             follow.delete()
 
-    log.info(f'Delete {count} Follow objects')
+    log.info(f"Delete {count} Follow objects")