udata 9.1.2.dev30355__py2.py3-none-any.whl → 9.1.2.dev30454__py2.py3-none-any.whl

udata/routing.py

@@ -1,21 +1,22 @@
-from bson import ObjectId
 from uuid import UUID
 
-from flask import request, redirect, url_for
+from bson import ObjectId
+from flask import redirect, request, url_for
 from mongoengine.errors import InvalidQueryError, ValidationError
 from werkzeug.exceptions import NotFound
 from werkzeug.routing import BaseConverter, PathConverter
 from werkzeug.urls import url_quote
 
 from udata import models
-from udata.mongo import db
-from udata.core.spatial.models import GeoZone
 from udata.core.dataservices.models import Dataservice
+from udata.core.spatial.models import GeoZone
 from udata.i18n import ISO_639_1_CODES
+from udata.mongo import db
 
 
 class LazyRedirect(object):
-    '''Store location for lazy redirections'''
+    """Store location for lazy redirections"""
+
     def __init__(self, arg):
         self.arg = arg
 
@@ -23,25 +24,23 @@ class LazyRedirect(object):
 class LanguagePrefixConverter(BaseConverter):
     def __init__(self, map):
         super(LanguagePrefixConverter, self).__init__(map)
-        self.regex = '(?:%s)' % '|'.join(ISO_639_1_CODES)
+        self.regex = "(?:%s)" % "|".join(ISO_639_1_CODES)
 
 
 class ListConverter(BaseConverter):
     def to_python(self, value):
-        return value.split(',')
+        return value.split(",")
 
     def to_url(self, values):
-        return ','.join(super(ListConverter, self).to_url(value)
-                        for value in values)
+        return ",".join(super(ListConverter, self).to_url(value) for value in values)
 
 
 class PathListConverter(PathConverter):
     def to_python(self, value):
-        return value.split(',')
+        return value.split(",")
 
     def to_url(self, values):
-        return ','.join(super(PathListConverter, self).to_url(value)
-                        for value in values)
+        return ",".join(super(PathListConverter, self).to_url(value) for value in values)
 
 
 class UUIDConverter(BaseConverter):
@@ -53,7 +52,7 @@ class UUIDConverter(BaseConverter):
 
 
 class ModelConverter(BaseConverter):
-    '''
+    """
     A base class helper for model helper.
 
     Allow to give model or slug or ObjectId as parameter to url_for().
@@ -66,13 +65,13 @@ class ModelConverter(BaseConverter):
     * fetch by id
     * fetch by slug
     * raise 404
-    '''
+    """
 
     model = None
 
     @property
     def has_slug(self):
-        return hasattr(self.model, 'slug') and isinstance(self.model.slug, db.SlugField)
+        return hasattr(self.model, "slug") and isinstance(self.model.slug, db.SlugField)
 
     @property
     def has_redirected_slug(self):
@@ -110,9 +109,9 @@ class ModelConverter(BaseConverter):
             return self.quote(obj)
         elif isinstance(obj, (ObjectId, UUID)):
             return str(obj)
-        elif getattr(obj, 'slug', None):
+        elif getattr(obj, "slug", None):
             return self.quote(obj.slug)
-        elif getattr(obj, 'id', None):
+        elif getattr(obj, "id", None):
             return str(obj.id)
         else:
             raise ValueError('Unable to serialize "%s" to url' % obj)
@@ -155,7 +154,7 @@ class ContactPointConverter(ModelConverter):
 
 
 class TerritoryConverter(PathConverter):
-    DEFAULT_PREFIX = 'fr'  # TODO: make it a setting parameter
+    DEFAULT_PREFIX = "fr"  # TODO: make it a setting parameter
 
     def to_python(self, value):
         """
@@ -166,10 +165,10 @@ class TerritoryConverter(PathConverter):
 
         Note that the slug is not significative but cannot be omitted.
         """
-        if '/' not in value:
+        if "/" not in value:
             return NotFound()
 
-        level, code = value.split('/')[:2]  # Ignore optional slug
+        level, code = value.split("/")[:2]  # Ignore optional slug
 
         geoid = GeoZone.SEPARATOR.join([level, code])
         zone = GeoZone.objects.resolve(geoid)
@@ -186,27 +185,23 @@ class TerritoryConverter(PathConverter):
         """
         Reconstruct the URL from level name, code or datagouv id and slug.
         """
-        level_name = getattr(obj, 'level_name', None)
+        level_name = getattr(obj, "level_name", None)
         if not level_name:
             raise ValueError('Unable to serialize "%s" to url' % obj)
 
-        code = getattr(obj, 'code', None)
-        slug = getattr(obj, 'slug', None)
+        code = getattr(obj, "code", None)
+        slug = getattr(obj, "slug", None)
         if code and slug:
-            return '{level_name}/{code}/{slug}'.format(
-                level_name=level_name,
-                code=code,
-                slug=slug
-            )
+            return "{level_name}/{code}/{slug}".format(level_name=level_name, code=code, slug=slug)
         else:
             raise ValueError('Unable to serialize "%s" to url' % obj)
 
 
 def lazy_raise_or_redirect():
-    '''
+    """
     Raise exception lazily to ensure request.endpoint is set
     Also perform redirect if needed
-    '''
+    """
     if not request.view_args:
         return
     for name, value in request.view_args.items():
@@ -217,22 +212,22 @@ def lazy_raise_or_redirect():
             new_args = request.view_args
             new_args[name] = value.arg
             new_url = url_for(request.endpoint, **new_args)
-            return redirect(new_url, code=204 if request.method == 'OPTIONS' else 308)
+            return redirect(new_url, 308)
 
 
 def init_app(app):
     app.before_request(lazy_raise_or_redirect)
-    app.url_map.converters['lang'] = LanguagePrefixConverter
-    app.url_map.converters['list'] = ListConverter
-    app.url_map.converters['pathlist'] = PathListConverter
-    app.url_map.converters['uuid'] = UUIDConverter
-    app.url_map.converters['dataset'] = DatasetConverter
-    app.url_map.converters['dataservice'] = DataserviceConverter
-    app.url_map.converters['crid'] = CommunityResourceConverter
-    app.url_map.converters['org'] = OrganizationConverter
-    app.url_map.converters['reuse'] = ReuseConverter
-    app.url_map.converters['user'] = UserConverter
-    app.url_map.converters['topic'] = TopicConverter
-    app.url_map.converters['post'] = PostConverter
-    app.url_map.converters['territory'] = TerritoryConverter
-    app.url_map.converters['contact_point'] = ContactPointConverter
+    app.url_map.converters["lang"] = LanguagePrefixConverter
+    app.url_map.converters["list"] = ListConverter
+    app.url_map.converters["pathlist"] = PathListConverter
+    app.url_map.converters["uuid"] = UUIDConverter
+    app.url_map.converters["dataset"] = DatasetConverter
+    app.url_map.converters["dataservice"] = DataserviceConverter
+    app.url_map.converters["crid"] = CommunityResourceConverter
+    app.url_map.converters["org"] = OrganizationConverter
+    app.url_map.converters["reuse"] = ReuseConverter
+    app.url_map.converters["user"] = UserConverter
+    app.url_map.converters["topic"] = TopicConverter
+    app.url_map.converters["post"] = PostConverter
+    app.url_map.converters["territory"] = TerritoryConverter
+    app.url_map.converters["contact_point"] = ContactPointConverter

udata/search/__init__.py

@@ -1,40 +1,39 @@
 import logging
+
 import requests
-import udata.event  # noqa
-# Import udata event in order for datasets event hooks to be executed
 
+# Import udata event in order for datasets event hooks to be executed
 from flask import current_app
-from mongoengine.signals import post_save, post_delete
+from mongoengine.signals import post_delete, post_save
 
+import udata.event  # noqa
 from udata.mongo import db
-from udata.tasks import task, as_task_param
+from udata.tasks import as_task_param, task
 
 log = logging.getLogger(__name__)
 
 adapter_catalog = {}
 
 
-@task(route='high.search')
+@task(route="high.search")
 def reindex(classname, id):
-    if not current_app.config['SEARCH_SERVICE_API_URL']:
+    if not current_app.config["SEARCH_SERVICE_API_URL"]:
         return
     model = db.resolve_model(classname)
     obj = model.objects.get(pk=id)
     adapter_class = adapter_catalog.get(model)
     document = adapter_class.serialize(obj)
     if adapter_class.is_indexable(obj):
-        log.info('Indexing %s (%s)', model.__name__, obj.id)
+        log.info("Indexing %s (%s)", model.__name__, obj.id)
         url = f"{current_app.config['SEARCH_SERVICE_API_URL']}{adapter_class.search_url}index"
         try:
-            payload = {
-                'document': document
-            }
+            payload = {"document": document}
             r = requests.post(url, json=payload)
             r.raise_for_status()
         except Exception:
             log.exception('Unable to index/unindex %s "%s"', model.__name__, str(obj.id))
     else:
-        log.info('Unindexing %s (%s)', model.__name__, obj.id)
+        log.info("Unindexing %s (%s)", model.__name__, obj.id)
         url = f"{current_app.config['SEARCH_SERVICE_API_URL']}{adapter_class.search_url}{str(obj.id)}/unindex"
         try:
             r = requests.delete(url)
@@ -46,13 +45,13 @@ def reindex(classname, id):
             log.exception('Unable to index/unindex %s "%s"', model.__name__, str(obj.id))
 
 
-@task(route='high.search')
+@task(route="high.search")
 def unindex(classname, id):
-    if not current_app.config['SEARCH_SERVICE_API_URL']:
+    if not current_app.config["SEARCH_SERVICE_API_URL"]:
         return
     model = db.resolve_model(classname)
     adapter_class = adapter_catalog.get(model)
-    log.info('Unindexing %s (%s)', model.__name__, id)
+    log.info("Unindexing %s (%s)", model.__name__, id)
     try:
         url = f"{current_app.config['SEARCH_SERVICE_API_URL']}{adapter_class.search_url}/{str(id)}/unindex"
         r = requests.delete(url)
@@ -65,19 +64,19 @@ def unindex(classname, id):
 
 
 def reindex_model_on_save(sender, document, **kwargs):
-    '''(Re/Un)Index Mongo document on post_save'''
-    if current_app.config.get('AUTO_INDEX') and current_app.config['SEARCH_SERVICE_API_URL']:
+    """(Re/Un)Index Mongo document on post_save"""
+    if current_app.config.get("AUTO_INDEX") and current_app.config["SEARCH_SERVICE_API_URL"]:
         reindex.delay(*as_task_param(document))
 
 
 def unindex_model_on_delete(sender, document, **kwargs):
-    '''Unindex Mongo document on post_delete'''
-    if current_app.config.get('AUTO_INDEX') and current_app.config['SEARCH_SERVICE_API_URL']:
+    """Unindex Mongo document on post_delete"""
+    if current_app.config.get("AUTO_INDEX") and current_app.config["SEARCH_SERVICE_API_URL"]:
         unindex.delay(*as_task_param(document))
 
 
 def register(adapter):
-    '''Register a search adapter'''
+    """Register a search adapter"""
     # register the class in the catalog
     if adapter.model and adapter.model not in adapter_catalog:
         adapter_catalog[adapter.model] = adapter

udata/search/adapter.py

@@ -1,13 +1,15 @@
 import logging
+
 from flask_restx.reqparse import RequestParser
-from udata.search.query import SearchQuery
 
+from udata.search.query import SearchQuery
 
 log = logging.getLogger(__name__)
 
 
 class ModelSearchAdapter:
     """This class allow to describe and customize the search behavior."""
+
     model = None
     sorts = None
     search_url = None
@@ -18,7 +20,7 @@ class ModelSearchAdapter:
         """By default use the ``to_dict`` method
         and exclude ``_id``, ``_cls`` and ``owner`` fields
         """
-        return document.to_dict(exclude=('_id', '_cls', 'owner'))
+        return document.to_dict(exclude=("_id", "_cls", "owner"))
 
     @classmethod
     def is_indexable(cls, document):
@@ -28,40 +30,39 @@ class ModelSearchAdapter:
     def as_request_parser(cls, paginate=True):
         parser = RequestParser()
         # q parameter
-        parser.add_argument('q', type=str, location='args',
-                            help='The search query')
+        parser.add_argument("q", type=str, location="args", help="The search query")
         # Add filters arguments
         for name, type in cls.filters.items():
             kwargs = type.as_request_parser_kwargs()
-            parser.add_argument(name, location='args', **kwargs)
+            parser.add_argument(name, location="args", **kwargs)
         # Sort arguments
         keys = list(cls.sorts)
-        choices = keys + ['-' + k for k in keys]
-        help_msg = 'The field (and direction) on which sorting apply'
-        parser.add_argument('sort', type=str, location='args', choices=choices,
-                            help=help_msg)
+        choices = keys + ["-" + k for k in keys]
+        help_msg = "The field (and direction) on which sorting apply"
+        parser.add_argument("sort", type=str, location="args", choices=choices, help=help_msg)
         if paginate:
-            parser.add_argument('page', type=int, location='args',
-                                default=1, help='The page to display')
-            parser.add_argument('page_size', type=int, location='args',
-                                default=20, help='The page size')
+            parser.add_argument(
+                "page", type=int, location="args", default=1, help="The page to display"
+            )
+            parser.add_argument(
+                "page_size", type=int, location="args", default=20, help="The page size"
+            )
         return parser
 
     @classmethod
     def parse_sort(cls, sort):
         if sort:
-            if sort.startswith('-'):
+            if sort.startswith("-"):
                 # Keyerror because of the '-' character in front of the argument.
                 # It is removed to find the value in dict and added back.
                 arg_sort = sort[1:]
-                sort = '-' + cls.sorts[arg_sort]
+                sort = "-" + cls.sorts[arg_sort]
             else:
                 sort = cls.sorts[sort]
         return sort
 
     @classmethod
     def temp_search(cls):
-
         class TempSearch(SearchQuery):
             adapter = cls
             model = cls.model

udata/search/commands.py

@@ -1,40 +1,39 @@
-from datetime import datetime
-from flask import current_app
 import logging
 import sys
-import requests
+from datetime import datetime
 
 import click
+import requests
+from flask import current_app
 
 from udata.commands import cli
 from udata.search import adapter_catalog
 
-
 log = logging.getLogger(__name__)
 
 
-@cli.group('search')
+@cli.group("search")
 def grp():
-    '''Search/Indexation related operations'''
+    """Search/Indexation related operations"""
     pass
 
 
-TIMESTAMP_FORMAT = '%Y-%m-%d-%H-%M'
+TIMESTAMP_FORMAT = "%Y-%m-%d-%H-%M"
 
 
 def default_index_suffix_name(now):
-    '''Build a time based index suffix name'''
+    """Build a time based index suffix name"""
     return now.strftime(TIMESTAMP_FORMAT)
 
 
 def iter_adapters():
-    '''Iter over adapter in predictable way'''
+    """Iter over adapter in predictable way"""
     adapters = adapter_catalog.values()
     return sorted(adapters, key=lambda a: a.model.__name__)
 
 
 def iter_qs(qs, adapter):
-    '''Safely iterate over a DB QuerySet yielding a tuple (indexability, serialized documents)'''
+    """Safely iterate over a DB QuerySet yielding a tuple (indexability, serialized documents)"""
     for obj in qs.no_cache().timeout(False):
         indexable = adapter.is_indexable(obj)
         try:
@@ -42,28 +41,25 @@ def iter_qs(qs, adapter):
             yield indexable, doc
         except Exception as e:
             model = adapter.model.__name__
-            log.error('Unable to index %s "%s": %s', model, str(obj.id),
-                      str(e), exc_info=True)
+            log.error('Unable to index %s "%s": %s', model, str(obj.id), str(e), exc_info=True)
 
 
 def index_model(adapter, start, reindex=False, from_datetime=None):
-    '''Index or unindex all objects given a model'''
+    """Index or unindex all objects given a model"""
     model = adapter.model
-    search_service_url = current_app.config['SEARCH_SERVICE_API_URL']
-    log.info('Indexing %s objects', model.__name__)
+    search_service_url = current_app.config["SEARCH_SERVICE_API_URL"]
+    log.info("Indexing %s objects", model.__name__)
     qs = model.objects
     if from_datetime:
-        date_property = ('last_modified_internal'
-                         if model.__name__.lower() in ['dataset']
-                         else 'last_modified')
-        qs = qs.filter(**{f'{date_property}__gte': from_datetime})
+        date_property = (
+            "last_modified_internal" if model.__name__.lower() in ["dataset"] else "last_modified"
+        )
+        qs = qs.filter(**{f"{date_property}__gte": from_datetime})
     model_name = adapter.model.__name__.lower()
     index_name = model_name
     if reindex:
-        index_name += '-' + default_index_suffix_name(start)
-        payload = {
-            'index': index_name
-        }
+        index_name += "-" + default_index_suffix_name(start)
+        payload = {"index": index_name}
         url = f"{search_service_url}/create-index"
         r = requests.post(url, json=payload)
         r.raise_for_status()
@@ -73,10 +69,7 @@ def index_model(adapter, start, reindex=False, from_datetime=None):
         with requests.Session() as session:
             try:
                 if indexable:
-                    payload = {
-                        'document': doc,
-                        'index': index_name
-                    }
+                    payload = {"document": doc, "index": index_name}
                     url = f"{search_service_url}/{model_name}s/index"
                     r = session.post(url, json=payload)
                     r.raise_for_status()
@@ -88,46 +81,46 @@ def index_model(adapter, start, reindex=False, from_datetime=None):
                 else:
                     continue
             except Exception as e:
-                log.error('Unable to index %s "%s": %s', model, str(doc['id']),
-                          str(e), exc_info=True)
+                log.error(
+                    'Unable to index %s "%s": %s', model, str(doc["id"]), str(e), exc_info=True
+                )
 
 
 def finalize_reindex(models, start):
     try:
         url = f"{current_app.config['SEARCH_SERVICE_API_URL']}/set-index-alias"
-        payload = {
-            'index_suffix_name': default_index_suffix_name(start),
-            'indices': models
-        }
+        payload = {"index_suffix_name": default_index_suffix_name(start), "indices": models}
         r = requests.post(url, json=payload)
         r.raise_for_status()
     except Exception:
-        log.exception(f'Unable to set alias for index')
+        log.exception(f"Unable to set alias for index")
 
     modified_since_reindex = 0
     for adapter in iter_adapters():
         if not models or adapter.model.__name__.lower() in models:
-            date_property = ('last_modified_internal'
-                             if adapter.model.__name__.lower() in ['dataset']
-                             else 'last_modified')
+            date_property = (
+                "last_modified_internal"
+                if adapter.model.__name__.lower() in ["dataset"]
+                else "last_modified"
+            )
             modified_since_reindex += adapter.model.objects(
-                **{f'{date_property}__gte': start}
+                **{f"{date_property}__gte": start}
             ).count()
 
     log.warning(
-        f'{modified_since_reindex} documents have been modified since reindexation start. '
-        f'After having set the appropriate alias, you can index last changes since the '
-        f'beginning of the indexation. Example, you can run:\n'
-        f'`udata search index -f {default_index_suffix_name(start)}`'
+        f"{modified_since_reindex} documents have been modified since reindexation start. "
+        f"After having set the appropriate alias, you can index last changes since the "
+        f"beginning of the indexation. Example, you can run:\n"
+        f"`udata search index -f {default_index_suffix_name(start)}`"
     )
 
 
 @grp.command()
-@click.argument('models', nargs=-1, metavar='[<model> ...]')
-@click.option('-r', '--reindex', default=False, type=bool)
-@click.option('-f', '--from_datetime', type=str)
+@click.argument("models", nargs=-1, metavar="[<model> ...]")
+@click.option("-r", "--reindex", default=False, type=bool)
+@click.option("-f", "--from_datetime", type=str)
 def index(models=None, reindex=True, from_datetime=None):
-    '''
+    """
     Initialize or rebuild the search index
 
     Models to reindex can optionally be specified as arguments.
@@ -136,9 +129,9 @@ def index(models=None, reindex=True, from_datetime=None):
     If reindex is true, indexation will be made on a new index and unindexable documents ignored.
 
     If from_datetime is specified, only models modified since this datetime will be indexed.
-    '''
-    if not current_app.config['SEARCH_SERVICE_API_URL']:
-        log.error('Missing URL for search service')
+    """
+    if not current_app.config["SEARCH_SERVICE_API_URL"]:
+        log.error("Missing URL for search service")
         sys.exit(-1)
 
     start = datetime.utcnow()
@@ -146,10 +139,10 @@ def index(models=None, reindex=True, from_datetime=None):
         from_datetime = datetime.strptime(from_datetime, TIMESTAMP_FORMAT)
 
     doc_types_names = [m.__name__.lower() for m in adapter_catalog.keys()]
-    models = [model.lower().rstrip('s') for model in (models or [])]
+    models = [model.lower().rstrip("s") for model in (models or [])]
     for model in models:
         if model not in doc_types_names:
-            log.error('Unknown model %s', model)
+            log.error("Unknown model %s", model)
             sys.exit(-1)
 
     for adapter in iter_adapters():

udata/search/fields.py

@@ -2,39 +2,36 @@ import logging
 import re
 
 from bson.objectid import ObjectId
-
 from flask_restx import inputs
 
 from udata.utils import clean_string
 
 log = logging.getLogger(__name__)
 
-__all__ = (
-    'BoolFilter', 'ModelTermsFilter', 'TemporalCoverageFilter', 'Filter'
-)
+__all__ = ("BoolFilter", "ModelTermsFilter", "TemporalCoverageFilter", "Filter")
 
 
-ES_NUM_FAILURES = '-Infinity', 'Infinity', 'NaN', None
+ES_NUM_FAILURES = "-Infinity", "Infinity", "NaN", None
 
-RE_TIME_COVERAGE = re.compile(r'\d{4}-\d{2}-\d{2}-\d{4}-\d{2}-\d{2}')
+RE_TIME_COVERAGE = re.compile(r"\d{4}-\d{2}-\d{2}-\d{4}-\d{2}-\d{2}")
 
-OR_SEPARATOR = '|'
+OR_SEPARATOR = "|"
 
 
 class Filter:
     @staticmethod
     def as_request_parser_kwargs():
-        return {'type': clean_string}
+        return {"type": clean_string}
 
 
 class BoolFilter(Filter):
     @staticmethod
     def as_request_parser_kwargs():
-        return {'type': inputs.boolean}
+        return {"type": inputs.boolean}
 
 
 class ModelTermsFilter(Filter):
-    def __init__(self, model, field_name='id'):
+    def __init__(self, model, field_name="id"):
         self.model = model
         self.field_name = field_name
 
@@ -46,10 +43,7 @@ class ModelTermsFilter(Filter):
         if isinstance(value, ObjectId):
             return value
         try:
-            return [
-                self.model_field.to_mongo(v)
-                for v in value.split(OR_SEPARATOR)
-            ]
+            return [self.model_field.to_mongo(v) for v in value.split(OR_SEPARATOR)]
         except Exception:
             raise ValueError('"{0}" is not valid identifier'.format(value))
 
@@ -57,8 +51,7 @@ class ModelTermsFilter(Filter):
 class TemporalCoverageFilter(Filter):
     @classmethod
     def validate_parameter(cls, value):
-        if not isinstance(value, str) \
-                or not RE_TIME_COVERAGE.match(value):
+        if not isinstance(value, str) or not RE_TIME_COVERAGE.match(value):
             msg = '"{0}" does not match YYYY-MM-DD-YYYY-MM-DD'.format(value)
             raise ValueError(msg)
         return value
@@ -66,8 +59,8 @@ class TemporalCoverageFilter(Filter):
     @classmethod
     def as_request_parser_kwargs(cls):
         return {
-            'type': cls.validate_parameter,
-            'help': 'A date range expressed as start-end '
-                    'where both dates are in iso format '
-                    '(ie. YYYY-MM-DD-YYYY-MM-DD)'
+            "type": cls.validate_parameter,
+            "help": "A date range expressed as start-end "
+            "where both dates are in iso format "
+            "(ie. YYYY-MM-DD-YYYY-MM-DD)",
         }