udata 9.1.2.dev30355__py2.py3-none-any.whl → 9.1.2.dev30454__py2.py3-none-any.whl

udata/core/dataset/search.py

@@ -1,122 +1,133 @@
 import datetime
-from udata.models import (
-    Dataset, Organization, User, GeoZone, License, Topic
-)
+
+from udata.core.dataset.api import DEFAULT_SORTING, DatasetApiParser
+from udata.core.spatial.constants import ADMIN_LEVEL_MAX
+from udata.core.spatial.models import admin_levels
+from udata.models import Dataset, GeoZone, License, Organization, Topic, User
 from udata.search import (
-    ModelSearchAdapter, register,
-    ModelTermsFilter, BoolFilter, Filter,
-    TemporalCoverageFilter
+    BoolFilter,
+    Filter,
+    ModelSearchAdapter,
+    ModelTermsFilter,
+    TemporalCoverageFilter,
+    register,
 )
-from udata.core.spatial.models import admin_levels
-from udata.core.spatial.constants import ADMIN_LEVEL_MAX
-from udata.core.dataset.api import DatasetApiParser, DEFAULT_SORTING
 from udata.utils import to_iso_datetime
 
-__all__ = ('DatasetSearch', )
+__all__ = ("DatasetSearch",)
 
 
 @register
 class DatasetSearch(ModelSearchAdapter):
     model = Dataset
-    search_url = 'datasets/'
+    search_url = "datasets/"
 
     sorts = {
-        'created': 'created_at_internal',
-        'last_update': 'last_modified_internal',
-        'reuses': 'metrics.reuses',
-        'followers': 'metrics.followers',
-        'views': 'metrics.views',
+        "created": "created_at_internal",
+        "last_update": "last_modified_internal",
+        "reuses": "metrics.reuses",
+        "followers": "metrics.followers",
+        "views": "metrics.views",
     }
 
     filters = {
-        'tag': Filter(),
-        'badge': Filter(),
-        'organization': ModelTermsFilter(model=Organization),
-        'owner': ModelTermsFilter(model=User),
-        'license': ModelTermsFilter(model=License),
-        'geozone': ModelTermsFilter(model=GeoZone),
-        'granularity': Filter(),
-        'format': Filter(),
-        'schema': Filter(),
-        'temporal_coverage': TemporalCoverageFilter(),
-        'featured': BoolFilter(),
-        'topic': ModelTermsFilter(model=Topic),
+        "tag": Filter(),
+        "badge": Filter(),
+        "organization": ModelTermsFilter(model=Organization),
+        "owner": ModelTermsFilter(model=User),
+        "license": ModelTermsFilter(model=License),
+        "geozone": ModelTermsFilter(model=GeoZone),
+        "granularity": Filter(),
+        "format": Filter(),
+        "schema": Filter(),
+        "temporal_coverage": TemporalCoverageFilter(),
+        "featured": BoolFilter(),
+        "topic": ModelTermsFilter(model=Topic),
     }
 
     @classmethod
     def is_indexable(cls, dataset):
-        return (dataset.deleted is None and dataset.archived is None and
-                not dataset.private)
+        return dataset.deleted is None and dataset.archived is None and not dataset.private
 
     @classmethod
     def mongo_search(cls, args):
         datasets = Dataset.objects(archived=None, deleted=None, private=False)
         datasets = DatasetApiParser.parse_filters(datasets, args)
 
-        sort = cls.parse_sort(args['sort']) or ('$text_score' if args['q'] else None) or DEFAULT_SORTING
-        offset = (args['page'] - 1) * args['page_size']
-        return datasets.order_by(sort).skip(offset).limit(args['page_size']), datasets.count()
+        sort = (
+            cls.parse_sort(args["sort"])
+            or ("$text_score" if args["q"] else None)
+            or DEFAULT_SORTING
+        )
+        offset = (args["page"] - 1) * args["page_size"]
+        return datasets.order_by(sort).skip(offset).limit(args["page_size"]), datasets.count()
 
     @classmethod
     def serialize(cls, dataset):
         organization = None
         owner = None
 
-        topics = Topic.objects(datasets=dataset).only('id')
+        topics = Topic.objects(datasets=dataset).only("id")
 
         if dataset.organization:
             org = Organization.objects(id=dataset.organization.id).first()
             organization = {
-                'id': str(org.id),
-                'name': org.name,
-                'public_service': 1 if org.public_service else 0,
-                'followers': org.metrics.get('followers', 0)
+                "id": str(org.id),
+                "name": org.name,
+                "public_service": 1 if org.public_service else 0,
+                "followers": org.metrics.get("followers", 0),
             }
         elif dataset.owner:
             owner = User.objects(id=dataset.owner.id).first()
 
         document = {
-            'id': str(dataset.id),
-            'title': dataset.title,
-            'description': dataset.description,
-            'acronym': dataset.acronym or None,
-            'url': dataset.display_url,
-            'tags': dataset.tags,
-            'license': getattr(dataset.license, 'id', None),
-            'badges': [badge.kind for badge in dataset.badges],
-            'frequency': dataset.frequency,
-            'created_at': to_iso_datetime(dataset.created_at),
-            'last_update': to_iso_datetime(dataset.last_update),
-            'views': dataset.metrics.get('views', 0),
-            'followers': dataset.metrics.get('followers', 0),
-            'reuses': dataset.metrics.get('reuses', 0),
-            'featured': 1 if dataset.featured else 0,
-            'resources_count': len(dataset.resources),
-            'organization': organization,
-            'owner': str(owner.id) if owner else None,
-            'format': [r.format.lower() for r in dataset.resources if r.format],
-            'schema': [r.schema.name for r in dataset.resources if r.schema],
-            'topics': [str(t.id) for t in topics if topics],
+            "id": str(dataset.id),
+            "title": dataset.title,
+            "description": dataset.description,
+            "acronym": dataset.acronym or None,
+            "url": dataset.display_url,
+            "tags": dataset.tags,
+            "license": getattr(dataset.license, "id", None),
+            "badges": [badge.kind for badge in dataset.badges],
+            "frequency": dataset.frequency,
+            "created_at": to_iso_datetime(dataset.created_at),
+            "last_update": to_iso_datetime(dataset.last_update),
+            "views": dataset.metrics.get("views", 0),
+            "followers": dataset.metrics.get("followers", 0),
+            "reuses": dataset.metrics.get("reuses", 0),
+            "featured": 1 if dataset.featured else 0,
+            "resources_count": len(dataset.resources),
+            "organization": organization,
+            "owner": str(owner.id) if owner else None,
+            "format": [r.format.lower() for r in dataset.resources if r.format],
+            "schema": [r.schema.name for r in dataset.resources if r.schema],
+            "topics": [str(t.id) for t in topics if topics],
         }
         extras = {}
         for key, value in dataset.extras.items():
             extras[key] = to_iso_datetime(value) if isinstance(value, datetime.datetime) else value
-        document.update({'extras': extras})
+        document.update({"extras": extras})
         if dataset.harvest:
             harvest = {}
             for key, value in dataset.harvest._data.items():
-                harvest[key] = to_iso_datetime(value) if isinstance(value, datetime.datetime) else value
-            document.update({'harvest': harvest})
+                harvest[key] = (
+                    to_iso_datetime(value) if isinstance(value, datetime.datetime) else value
+                )
+            document.update({"harvest": harvest})
 
-        if (dataset.temporal_coverage is not None and
-                dataset.temporal_coverage.start and
-                dataset.temporal_coverage.end):
+        if (
+            dataset.temporal_coverage is not None
+            and dataset.temporal_coverage.start
+            and dataset.temporal_coverage.end
+        ):
             start = to_iso_datetime(dataset.temporal_coverage.start)
             end = to_iso_datetime(dataset.temporal_coverage.end)
-            document.update({
-                'temporal_coverage_start': start,
-                'temporal_coverage_end': end,
-            })
+            document.update(
+                {
+                    "temporal_coverage_start": start,
+                    "temporal_coverage_end": end,
+                }
+            )
 
         if dataset.spatial is not None:
             # Index precise zone labels to allow fast filtering.
@@ -125,13 +136,17 @@ class DatasetSearch(ModelSearchAdapter):
             geozones = []
             coverage_level = ADMIN_LEVEL_MAX
             for zone in zones:
-                geozones.append({
-                    'id': zone.id,
-                    'name': zone.name,
-                })
+                geozones.append(
+                    {
+                        "id": zone.id,
+                        "name": zone.name,
+                    }
+                )
                 coverage_level = min(coverage_level, admin_levels[zone.level])
-            document.update({
-                'geozones': geozones,
-                'granularity': dataset.spatial.granularity,
-            })
+            document.update(
+                {
+                    "geozones": geozones,
+                    "granularity": dataset.spatial.granularity,
+                }
+            )
         return document

udata/core/dataset/signals.py

@@ -3,4 +3,4 @@ from blinker import Namespace
 namespace = Namespace()
 
 #: Trigerred when a dataset is published
-on_dataset_published = namespace.signal('on-dataset-published')
+on_dataset_published = namespace.signal("on-dataset-published")

udata/core/dataset/tasks.py

@@ -1,6 +1,5 @@
 import collections
 import os
-
 from datetime import datetime, timedelta
 from tempfile import NamedTemporaryFile
 
@@ -13,28 +12,29 @@ from udata.core import storages
 from udata.frontend import csv
 from udata.harvest.models import HarvestJob
 from udata.i18n import lazy_gettext as _
-from udata.models import (Follow, Discussion, Activity, Topic,
-                          Organization, Transfer, db)
+from udata.models import Activity, Discussion, Follow, Organization, Topic, Transfer, db
 from udata.tasks import job
 
-from .models import Dataset, Resource, CommunityResource, Checksum
 from .constants import UPDATE_FREQUENCIES
+from .models import Checksum, CommunityResource, Dataset, Resource
 
 log = get_task_logger(__name__)
 
 
 def flatten(iterable):
     for el in iterable:
-        if isinstance(el, collections.Iterable) and not (isinstance(el, str) or isinstance(el, db.Document)):
+        if isinstance(el, collections.Iterable) and not (
+            isinstance(el, str) or isinstance(el, db.Document)
+        ):
             yield from flatten(el)
         else:
             yield el
 
 
-@job('purge-datasets')
+@job("purge-datasets")
 def purge_datasets(self):
     for dataset in Dataset.objects(deleted__ne=None):
-        log.info(f'Purging dataset {dataset}')
+        log.info(f"Purging dataset {dataset}")
         # Remove followers
         Follow.objects(following=dataset).delete()
         # Remove discussions
@@ -72,19 +72,23 @@ def purge_datasets(self):
         dataset.delete()
 
 
-@job('send-frequency-reminder')
+@job("send-frequency-reminder")
 def send_frequency_reminder(self):
     # We exclude irrelevant frequencies.
-    frequencies = [f for f in UPDATE_FREQUENCIES.keys()
-                   if f not in ('unknown', 'realtime', 'punctual', 'irregular', 'continuous')]
+    frequencies = [
+        f
+        for f in UPDATE_FREQUENCIES.keys()
+        if f not in ("unknown", "realtime", "punctual", "irregular", "continuous")
+    ]
     now = datetime.utcnow()
     reminded_orgs = {}
     reminded_people = []
-    allowed_delay = current_app.config['DELAY_BEFORE_REMINDER_NOTIFICATION']
+    allowed_delay = current_app.config["DELAY_BEFORE_REMINDER_NOTIFICATION"]
     for org in Organization.objects.visible():
         outdated_datasets = []
         for dataset in Dataset.objects.filter(
-                frequency__in=frequencies, organization=org).visible():
+            frequency__in=frequencies, organization=org
+        ).visible():
             if dataset.next_update + timedelta(days=allowed_delay) < now:
                 dataset.outdated = now - dataset.next_update
                 dataset.frequency_str = UPDATE_FREQUENCIES[dataset.frequency]
@@ -92,23 +96,28 @@ def send_frequency_reminder(self):
         if outdated_datasets:
             reminded_orgs[org] = outdated_datasets
     for reminded_org, datasets in reminded_orgs.items():
-        print('{org.name} will be emailed for {datasets_nb} datasets'.format(
-              org=reminded_org, datasets_nb=len(datasets)))
+        print(
+            "{org.name} will be emailed for {datasets_nb} datasets".format(
+                org=reminded_org, datasets_nb=len(datasets)
+            )
+        )
         recipients = [m.user for m in reminded_org.members]
         reminded_people.append(recipients)
-        subject = _('You need to update some frequency-based datasets')
-        mail.send(subject, recipients, 'frequency_reminder',
-                  org=reminded_org, datasets=datasets)
+        subject = _("You need to update some frequency-based datasets")
+        mail.send(subject, recipients, "frequency_reminder", org=reminded_org, datasets=datasets)
 
-    print('{nb_orgs} orgs concerned'.format(nb_orgs=len(reminded_orgs)))
+    print("{nb_orgs} orgs concerned".format(nb_orgs=len(reminded_orgs)))
     reminded_people = list(flatten(reminded_people))
-    print('{nb_emails} people contacted ({nb_emails_twice} twice)'.format(
-        nb_emails=len(reminded_people),
-        nb_emails_twice=len(reminded_people) - len(set(reminded_people))))
-    print('Done')
+    print(
+        "{nb_emails} people contacted ({nb_emails_twice} twice)".format(
+            nb_emails=len(reminded_people),
+            nb_emails_twice=len(reminded_people) - len(set(reminded_people)),
+        )
+    )
+    print("Done")
 
 
-@job('update-datasets-reuses-metrics')
+@job("update-datasets-reuses-metrics")
 def update_datasets_reuses_metrics(self):
     all_datasets = Dataset.objects.visible().timeout(False)
     for dataset in all_datasets:
@@ -117,10 +126,10 @@ def update_datasets_reuses_metrics(self):
 
 def get_queryset(model_cls):
     # special case for resources
-    if model_cls.__name__ == 'Resource':
-        model_cls = getattr(udata_models, 'Dataset')
+    if model_cls.__name__ == "Resource":
+        model_cls = getattr(udata_models, "Dataset")
     params = {}
-    attrs = ('private', 'deleted')
+    attrs = ("private", "deleted")
     for attr in attrs:
         if getattr(model_cls, attr, None):
             params[attr] = False
@@ -131,7 +140,7 @@ def get_queryset(model_cls):
 def get_or_create_resource(r_info, model, dataset):
     resource = None
     for r in dataset.resources:
-        if r.extras.get('csv-export:model', '') == model:
+        if r.extras.get("csv-export:model", "") == model:
             resource = r
             break
     if resource:
@@ -140,45 +149,45 @@ def get_or_create_resource(r_info, model, dataset):
         resource.save()
         return False, resource
     else:
-        r_info['extras'] = {'csv-export:model': model}
+        r_info["extras"] = {"csv-export:model": model}
         return True, Resource(**r_info)
 
 
 def store_resource(csvfile, model, dataset):
-    timestr = datetime.utcnow().strftime('%Y%m%d-%H%M%S')
-    filename = 'export-%s-%s.csv' % (model, timestr)
-    prefix = '/'.join((dataset.slug, timestr))
+    timestr = datetime.utcnow().strftime("%Y%m%d-%H%M%S")
+    filename = "export-%s-%s.csv" % (model, timestr)
+    prefix = "/".join((dataset.slug, timestr))
     storage = storages.resources
-    with open(csvfile.name, 'rb') as infile:
+    with open(csvfile.name, "rb") as infile:
         stored_filename = storage.save(infile, prefix=prefix, filename=filename)
     r_info = storage.metadata(stored_filename)
-    r_info['last_modified_internal'] = r_info.pop('modified')
-    r_info['fs_filename'] = stored_filename
-    checksum = r_info.pop('checksum')
-    algo, checksum = checksum.split(':', 1)
-    r_info['format'] = storages.utils.extension(stored_filename)
-    r_info['checksum'] = Checksum(type=algo, value=checksum)
-    r_info['filesize'] = r_info.pop('size')
-    del r_info['filename']
-    r_info['title'] = filename
+    r_info["last_modified_internal"] = r_info.pop("modified")
+    r_info["fs_filename"] = stored_filename
+    checksum = r_info.pop("checksum")
+    algo, checksum = checksum.split(":", 1)
+    r_info["format"] = storages.utils.extension(stored_filename)
+    r_info["checksum"] = Checksum(type=algo, value=checksum)
+    r_info["filesize"] = r_info.pop("size")
+    del r_info["filename"]
+    r_info["title"] = filename
     return get_or_create_resource(r_info, model, dataset)
 
 
 def export_csv_for_model(model, dataset):
     model_cls = getattr(udata_models, model.capitalize(), None)
     if not model_cls:
-        log.error('Unknow model %s' % model)
+        log.error("Unknow model %s" % model)
         return
     queryset = get_queryset(model_cls)
     adapter = csv.get_adapter(model_cls)
     if not adapter:
-        log.error('No adapter found for %s' % model)
+        log.error("No adapter found for %s" % model)
         return
     adapter = adapter(queryset)
 
-    log.info('Exporting CSV for %s...' % model)
+    log.info("Exporting CSV for %s..." % model)
 
-    csvfile = NamedTemporaryFile(mode='w', encoding='utf8', delete=False)
+    csvfile = NamedTemporaryFile(mode="w", encoding="utf8", delete=False)
     try:
         # write adapter results into a tmp file
         writer = csv.get_writer(csvfile)
@@ -198,27 +207,27 @@ def export_csv_for_model(model, dataset):
         os.unlink(csvfile.name)
 
 
-@job('export-csv')
+@job("export-csv")
 def export_csv(self, model=None):
-    '''
+    """
     Generates a CSV export of all model objects as a resource of a dataset
-    '''
-    ALLOWED_MODELS = current_app.config.get('EXPORT_CSV_MODELS', [])
-    DATASET_ID = current_app.config.get('EXPORT_CSV_DATASET_ID')
+    """
+    ALLOWED_MODELS = current_app.config.get("EXPORT_CSV_MODELS", [])
+    DATASET_ID = current_app.config.get("EXPORT_CSV_DATASET_ID")
 
     if model and model not in ALLOWED_MODELS:
-        log.error('Unknown or unallowed model: %s' % model)
+        log.error("Unknown or unallowed model: %s" % model)
         return
 
     if not DATASET_ID:
-        log.error('EXPORT_CSV_DATASET_ID setting value not set')
+        log.error("EXPORT_CSV_DATASET_ID setting value not set")
         return
     try:
         dataset = Dataset.objects.get(id=DATASET_ID)
     except Dataset.DoesNotExist:
-        log.error('EXPORT_CSV_DATASET_ID points to a non existent dataset')
+        log.error("EXPORT_CSV_DATASET_ID points to a non existent dataset")
         return
 
-    models = (model, ) if model else ALLOWED_MODELS
+    models = (model,) if model else ALLOWED_MODELS
     for model in models:
         export_csv_for_model(model, dataset)

udata/core/discussions/actions.py

@@ -1,20 +1,20 @@
-from udata.models import Reuse, Dataset
+from udata.models import Dataset, Reuse
 
 from .models import Discussion
 
 
 def discussions_for(user, only_open=True):
-    '''
+    """
     Build a queryset to query discussions related to a given user's assets.
 
     It includes discussions coming from the user's organizations
 
     :param bool only_open: whether to include closed discussions or not.
-    '''
+    """
     # Only fetch required fields for discussion filtering (id and slug)
     # Greatly improve performances and memory usage
-    datasets = Dataset.objects.owned_by(user.id, *user.organizations).only('id', 'slug')
-    reuses = Reuse.objects.owned_by(user.id, *user.organizations).only('id', 'slug')
+    datasets = Dataset.objects.owned_by(user.id, *user.organizations).only("id", "slug")
+    reuses = Reuse.objects.owned_by(user.id, *user.organizations).only("id", "slug")
 
     qs = Discussion.objects(subject__in=list(datasets) + list(reuses))
     if only_open: