udata 9.1.2.dev30355__py2.py3-none-any.whl → 9.1.2.dev30454__py2.py3-none-any.whl

udata/core/spatial/commands.py

@@ -2,16 +2,14 @@ import json
 import logging
 import signal
 import sys
-
 from collections import Counter
 from contextlib import contextmanager
 from datetime import datetime
 from textwrap import dedent
-import requests
 
 import click
+import requests
 import slugify
-
 from mongoengine import errors
 from mongoengine.context_managers import switch_collection
 
@@ -23,22 +21,22 @@ from udata.core.spatial.models import GeoLevel, GeoZone, SpatialCoverage
 log = logging.getLogger(__name__)
 
 
-DEFAULT_GEOZONES_FILE = 'https://www.data.gouv.fr/fr/datasets/r/a1bb263a-6cc7-4871-ab4f-2470235a67bf'
-DEFAULT_LEVELS_FILE = 'https://www.data.gouv.fr/fr/datasets/r/e0206442-78b3-4a00-b71c-c065d20561c8'
+DEFAULT_GEOZONES_FILE = (
+    "https://www.data.gouv.fr/fr/datasets/r/a1bb263a-6cc7-4871-ab4f-2470235a67bf"
+)
+DEFAULT_LEVELS_FILE = "https://www.data.gouv.fr/fr/datasets/r/e0206442-78b3-4a00-b71c-c065d20561c8"
 
 
-@cli.group('spatial')
+@cli.group("spatial")
 def grp():
-    '''Geospatial related operations'''
+    """Geospatial related operations"""
     pass
 
 
 def load_levels(col, json_levels):
     for i, level in enumerate(json_levels):
-        col.objects(id=level['id']).modify(
-            upsert=True,
-            set__name=level['label'],
-            set__admin_level=level.get('admin_level')
+        col.objects(id=level["id"]).modify(
+            upsert=True, set__name=level["label"], set__admin_level=level.get("admin_level")
         )
     return i
 
@@ -46,72 +44,71 @@ def load_levels(col, json_levels):
 def load_zones(col, json_geozones):
     loaded_geozones = 0
     for _, geozone in enumerate(json_geozones):
-        if geozone.get('is_deleted', False):
+        if geozone.get("is_deleted", False):
             continue
         params = {
-            'slug': slugify.slugify(geozone['nom'], separator='-'),
-            'level': str(geozone['level']),
-            'code': geozone['codeINSEE'],
-            'name': geozone['nom'],
-            'uri': geozone['uri']
+            "slug": slugify.slugify(geozone["nom"], separator="-"),
+            "level": str(geozone["level"]),
+            "code": geozone["codeINSEE"],
+            "name": geozone["nom"],
+            "uri": geozone["uri"],
         }
         try:
-            col.objects(id=geozone['_id']).modify(upsert=True, **{
-                'set__{0}'.format(k): v for k, v in params.items()
-            })
+            col.objects(id=geozone["_id"]).modify(
+                upsert=True, **{"set__{0}".format(k): v for k, v in params.items()}
+            )
             loaded_geozones += 1
         except errors.ValidationError as e:
-            log.warning('Validation error (%s) for %s with %s',
-                        e, geozone['nom'], params)
+            log.warning("Validation error (%s) for %s with %s", e, geozone["nom"], params)
             continue
     return loaded_geozones
 
 
 @contextmanager
 def handle_error(to_delete=None):
-    '''
+    """
     Handle errors while loading.
     In case of error, properly log it, remove the temporary files and collections and exit.
     If `to_delete` is given a collection, it will be deleted deleted.
-    '''
+    """
     # Handle keyboard interrupt
     signal.signal(signal.SIGINT, signal.default_int_handler)
     signal.signal(signal.SIGTERM, signal.default_int_handler)
     try:
         yield
     except KeyboardInterrupt:
-        print('')  # Proper warning message under the "^C" display
-        log.warning('Interrupted by signal')
+        print("")  # Proper warning message under the "^C" display
+        log.warning("Interrupted by signal")
     except Exception as e:
         log.error(e)
     else:
         return  # Nothing to do in case of success
     if to_delete:
-        log.info('Removing temporary collection %s', to_delete._get_collection_name())
+        log.info("Removing temporary collection %s", to_delete._get_collection_name())
         to_delete.drop_collection()
     sys.exit(-1)
 
 
 @grp.command()
-@click.argument('geozones-file', default=DEFAULT_GEOZONES_FILE)
-@click.argument('levels-file', default=DEFAULT_LEVELS_FILE)
-@click.option('-d', '--drop', is_flag=True, help='Drop existing data')
+@click.argument("geozones-file", default=DEFAULT_GEOZONES_FILE)
+@click.argument("levels-file", default=DEFAULT_LEVELS_FILE)
+@click.option("-d", "--drop", is_flag=True, help="Drop existing data")
 def load(geozones_file, levels_file, drop=False):
-    '''
+    """
     Load a geozones archive from <filename>
 
     <filename> can be either a local path or a remote URL.
-    '''
-    log.info('Loading GeoZones levels')
-    if levels_file.startswith('http'):
+    """
+    log.info("Loading GeoZones levels")
+    if levels_file.startswith("http"):
         json_levels = requests.get(levels_file).json()
     else:
         with open(levels_file) as f:
             json_levels = json.load(f)
 
-    ts = datetime.utcnow().isoformat().replace('-', '').replace(':', '').split('.')[0]
+    ts = datetime.utcnow().isoformat().replace("-", "").replace(":", "").split(".")[0]
     if drop and GeoLevel.objects.count():
-        name = '_'.join((GeoLevel._get_collection_name(), ts))
+        name = "_".join((GeoLevel._get_collection_name(), ts))
         target = GeoLevel._get_collection_name()
         with switch_collection(GeoLevel, name):
             with handle_error(GeoLevel):
@@ -120,17 +117,17 @@ def load(geozones_file, levels_file, drop=False):
     else:
         with handle_error():
             total = load_levels(GeoLevel, json_levels)
-    log.info('Loaded {total} levels'.format(total=total))
+    log.info("Loaded {total} levels".format(total=total))
 
-    log.info('Loading Zones')
-    if geozones_file.startswith('http'):
+    log.info("Loading Zones")
+    if geozones_file.startswith("http"):
         json_geozones = requests.get(geozones_file).json()
     else:
         with open(geozones_file) as f:
             json_geozones = json.load(f)
 
     if drop and GeoZone.objects.count():
-        name = '_'.join((GeoZone._get_collection_name(), ts))
+        name = "_".join((GeoZone._get_collection_name(), ts))
         target = GeoZone._get_collection_name()
         with switch_collection(GeoZone, name):
             with handle_error(GeoZone):
@@ -139,35 +136,35 @@ def load(geozones_file, levels_file, drop=False):
     else:
         with handle_error():
             total = load_zones(GeoZone, json_geozones)
-    log.info('Loaded {total} zones'.format(total=total))
+    log.info("Loaded {total} zones".format(total=total))
 
-    log.info('Clean removed geozones in datasets')
+    log.info("Clean removed geozones in datasets")
     count = fixup_removed_geozone()
-    log.info(f'{count} geozones removed from datasets')
+    log.info(f"{count} geozones removed from datasets")
 
 
 @grp.command()
 def migrate():
-    '''
+    """
     Migrate zones from old to new ids in datasets.
 
     Should only be run once with the new version of geozones w/ geohisto.
-    '''
-    counter = Counter(['zones', 'datasets'])
-    qs = GeoZone.objects.only('id', 'level')
+    """
+    counter = Counter(["zones", "datasets"])
+    qs = GeoZone.objects.only("id", "level")
     # Fetch datasets with non-empty spatial zones
     for dataset in Dataset.objects(spatial__zones__gt=[]):
-        counter['datasets'] += 1
+        counter["datasets"] += 1
         new_zones = []
         for current_zone in dataset.spatial.zones:
-            counter['zones'] += 1
+            counter["zones"] += 1
 
             level, code = geoids.parse(current_zone.id)
             zone = qs(level=level, code=code).first() or qs(code=code).first()
 
             if not zone:
-                log.warning('No match for %s: skipped', current_zone.id)
-                counter['skipped'] += 1
+                log.warning("No match for %s: skipped", current_zone.id)
+                counter["skipped"] += 1
                 continue
 
             new_zones.append(zone.id)
@@ -175,30 +172,37 @@ def migrate():
 
         # Update dataset with new spatial zones
         dataset.update(
-            spatial=SpatialCoverage(
-                granularity=dataset.spatial.granularity,
-                zones=list(new_zones)
-            )
+            spatial=SpatialCoverage(granularity=dataset.spatial.granularity, zones=list(new_zones))
         )
 
-    level_summary = '\n'.join([
-        ' - {0}: {1}'.format(l.id, counter[l.id])
-        for l in GeoLevel.objects.order_by('admin_level')
-    ])
-    summary = '\n'.join([dedent('''\
+    level_summary = "\n".join(
+        [
+            " - {0}: {1}".format(l.id, counter[l.id])
+            for l in GeoLevel.objects.order_by("admin_level")
+        ]
+    )
+    summary = "\n".join(
+        [
+            dedent(
+                """\
     Summary
     =======
     Processed {zones} zones in {datasets} datasets:\
-    '''.format(level_summary, **counter)), level_summary])
+    """.format(level_summary, **counter)
+            ),
+            level_summary,
+        ]
+    )
     log.info(summary)
-    log.info('Done')
+    log.info("Done")
+
 
 def fixup_removed_geozone():
     count = 0
     all_datasets = Dataset.objects(spatial__zones__0__exists=True).timeout(False)
     for dataset in all_datasets:
         zones = dataset.spatial.zones
-        new_zones = [z for z in zones if getattr(z, 'name', None) is not None]
+        new_zones = [z for z in zones if getattr(z, "name", None) is not None]
 
         if len(new_zones) < len(zones):
             log.debug(f"Removing deleted zones from dataset '{dataset.title}'")
@@ -207,4 +211,3 @@ def fixup_removed_geozone():
             dataset.save()
 
     return count
-        

udata/core/spatial/constants.py

@@ -1,9 +1,9 @@
 from udata.i18n import L_
 
 BASE_GRANULARITIES = [
-    ('poi', L_('POI')),
-    ('other', L_('Other')),
+    ("poi", L_("POI")),
+    ("other", L_("Other")),
 ]
 
 ADMIN_LEVEL_MIN = 1
-ADMIN_LEVEL_MAX = 110
+ADMIN_LEVEL_MAX = 110

udata/core/spatial/factories.py

@@ -1,7 +1,5 @@
 import factory
-
 from faker.providers import BaseProvider
-
 from geojson.utils import generate_random
 
 from udata.factories import DateRangeFactory, ModelFactory
@@ -13,90 +11,75 @@ from .models import GeoLevel, GeoZone, SpatialCoverage, spatial_granularities
 
 @faker_provider
 class GeoJsonProvider(BaseProvider):
-    '''A Fake GeoJSON provider'''
+    """A Fake GeoJSON provider"""
 
     def random_range(self, min=2, max=5):
         return range(self.random_int(min, max))
 
     def point(self):
-        return generate_random('Point')
+        return generate_random("Point")
 
     def linestring(self):
-        return generate_random('LineString')
+        return generate_random("LineString")
 
     def polygon(self):
-        return generate_random('Polygon')
+        return generate_random("Polygon")
 
     def multipoint(self):
-        coordinates = [
-            generate_random('Point')['coordinates']
-            for _ in self.random_range()
-        ]
+        coordinates = [generate_random("Point")["coordinates"] for _ in self.random_range()]
 
-        return {
-            'type': 'MultiPoint',
-            'coordinates': coordinates
-        }
+        return {"type": "MultiPoint", "coordinates": coordinates}
 
     def multilinestring(self):
-        coordinates = [
-            generate_random('LineString')['coordinates']
-            for _ in self.random_range()
-        ]
+        coordinates = [generate_random("LineString")["coordinates"] for _ in self.random_range()]
 
-        return {
-            'type': 'MultiLineString',
-            'coordinates': coordinates
-        }
+        return {"type": "MultiLineString", "coordinates": coordinates}
 
     def multipolygon(self):
-        coordinates = [
-            generate_random('Polygon')['coordinates']
-            for _ in self.random_range()
-        ]
+        coordinates = [generate_random("Polygon")["coordinates"] for _ in self.random_range()]
 
-        return {
-            'type': 'MultiPolygon',
-            'coordinates': coordinates
-        }
+        return {"type": "MultiPolygon", "coordinates": coordinates}
 
     def geometry_collection(self):
         element_factories = [
-            self.point, self.linestring, self.polygon,
-            self.multipoint, self.multilinestring, self.multipolygon
+            self.point,
+            self.linestring,
+            self.polygon,
+            self.multipoint,
+            self.multilinestring,
+            self.multipolygon,
         ]
         return {
-            'type': 'GeometryCollection',
-            'geometries': [
-                self.random_element(element_factories)()
-                for _ in self.random_range()
-            ]
+            "type": "GeometryCollection",
+            "geometries": [self.random_element(element_factories)() for _ in self.random_range()],
         }
 
     def feature(self):
         element_factories = [
-            self.point, self.linestring, self.polygon,
-            self.multipoint, self.multilinestring, self.multipolygon
+            self.point,
+            self.linestring,
+            self.polygon,
+            self.multipoint,
+            self.multilinestring,
+            self.multipolygon,
         ]
         return {
-            'type': 'Feature',
-            'geometry': self.random_element(element_factories)(),
-            'properties': {},
+            "type": "Feature",
+            "geometry": self.random_element(element_factories)(),
+            "properties": {},
         }
 
     def feature_collection(self):
         return {
-            'type': 'FeatureCollection',
-            'features': [self.feature() for _ in self.random_range()]
+            "type": "FeatureCollection",
+            "features": [self.feature() for _ in self.random_range()],
         }
 
 
 @faker_provider
 class SpatialProvider(BaseProvider):
     def spatial_granularity(self):
-        return self.generator.random_element([
-            row[0] for row in spatial_granularities
-        ])
+        return self.generator.random_element([row[0] for row in spatial_granularities])
 
 
 class GeoZoneFactory(ModelFactory):
@@ -104,10 +87,10 @@ class GeoZoneFactory(ModelFactory):
         model = GeoZone
 
     id = factory.LazyAttribute(geoids.from_zone)
-    name = factory.Faker('city')
-    slug = factory.Faker('slug')
-    code = factory.Faker('zipcode')
-    uri = factory.Faker('url')
+    name = factory.Faker("city")
+    slug = factory.Faker("slug")
+    code = factory.Faker("zipcode")
+    uri = factory.Faker("url")
     level = factory.LazyAttribute(lambda o: GeoLevelFactory().id)
 
 
@@ -116,12 +99,12 @@ class SpatialCoverageFactory(ModelFactory):
         model = SpatialCoverage
 
     zones = factory.LazyAttribute(lambda o: [GeoZoneFactory()])
-    granularity = factory.Faker('spatial_granularity')
+    granularity = factory.Faker("spatial_granularity")
 
 
 class GeoLevelFactory(ModelFactory):
     class Meta:
         model = GeoLevel
 
-    id = factory.Faker('unique_string')
-    name = factory.Faker('name')
+    id = factory.Faker("unique_string")
+    name = factory.Faker("name")

udata/core/spatial/forms.py

@@ -1,9 +1,10 @@
-import geojson
 import json
 import logging
 
-from udata.forms import widgets, ModelForm, validators
-from udata.forms.fields import ModelList, Field, SelectField, FormField
+import geojson
+
+from udata.forms import ModelForm, validators, widgets
+from udata.forms.fields import Field, FormField, ModelList, SelectField
 from udata.i18n import lazy_gettext as _
 
 from .models import GeoZone, SpatialCoverage, spatial_granularities
@@ -12,15 +13,14 @@ log = logging.getLogger(__name__)
 
 
 class ZonesAutocompleter(widgets.TextInput):
-    classes = 'zone-completer'
+    classes = "zone-completer"
 
     def __call__(self, field, **kwargs):
-        '''Store the values as JSON to prefeed selectize'''
+        """Store the values as JSON to prefeed selectize"""
         if field.data:
-            kwargs['data-values'] = json.dumps([{
-                'id': zone.id,
-                'name': zone.name
-            } for zone in field.data])
+            kwargs["data-values"] = json.dumps(
+                [{"id": zone.id, "name": zone.name} for zone in field.data]
+            )
         return super(ZonesAutocompleter, self).__call__(field, **kwargs)
 
 
@@ -29,12 +29,12 @@ class ZonesField(ModelList, Field):
     widget = ZonesAutocompleter()
 
     def fetch_objects(self, geoids):
-        '''
+        """
         Custom object retrieval.
 
         Zones are resolved from their identifier
         instead of the default bulk fetch by ID.
-        '''
+        """
         zones = []
         no_match = []
         for geoid in geoids:
@@ -45,8 +45,9 @@ class ZonesField(ModelList, Field):
                 no_match.append(geoid)
 
         if no_match:
-            msg = _('Unknown geoid(s): {identifiers}').format(
-                identifiers=', '.join(str(id) for id in no_match))
+            msg = _("Unknown geoid(s): {identifiers}").format(
+                identifiers=", ".join(str(id) for id in no_match)
+            )
             raise validators.ValidationError(msg)
 
         return zones
@@ -63,15 +64,15 @@ class GeomField(Field):
                     self.data = geojson.GeoJSON.to_instance(value)
             except:
                 self.data = None
-                log.exception('Unable to parse GeoJSON')
-                raise ValueError(self.gettext('Not a valid GeoJSON'))
+                log.exception("Unable to parse GeoJSON")
+                raise ValueError(self.gettext("Not a valid GeoJSON"))
 
     def pre_validate(self, form):
         if self.data:
             if not isinstance(self.data, geojson.GeoJSON):
                 self.data = geojson.GeoJSON.to_instance(self.data)
                 if not isinstance(self.data, geojson.GeoJSON):
-                    raise validators.ValidationError('Not a valid GeoJSON')
+                    raise validators.ValidationError("Not a valid GeoJSON")
             if not self.data.is_valid:
                 raise validators.ValidationError(self.data.errors())
         return True
@@ -80,18 +81,18 @@ class GeomField(Field):
 class SpatialCoverageForm(ModelForm):
     model_class = SpatialCoverage
 
-    zones = ZonesField(_('Spatial coverage'),
-                       description=_('A list of covered territories'),
-                       default=[])
-    granularity = SelectField(_('Spatial granularity'),
-                              description=_('The size of the data increment'),
-                              choices=lambda: spatial_granularities,
-                              default='other')
+    zones = ZonesField(
+        _("Spatial coverage"), description=_("A list of covered territories"), default=[]
+    )
+    granularity = SelectField(
+        _("Spatial granularity"),
+        description=_("The size of the data increment"),
+        choices=lambda: spatial_granularities,
+        default="other",
+    )
     geom = GeomField()
 
 
 class SpatialCoverageField(FormField):
     def __init__(self, *args, **kwargs):
-        super(SpatialCoverageField, self).__init__(SpatialCoverageForm,
-                                                   *args,
-                                                   **kwargs)
+        super(SpatialCoverageField, self).__init__(SpatialCoverageForm, *args, **kwargs)

udata/core/spatial/geoids.py

@@ -1,42 +1,42 @@
-'''
+"""
 This module centralize GeoID resources and helpers.
 
 See https://github.com/etalab/geoids for more details
-'''
+"""
 
-
-__all__ = ('GeoIDError', 'parse', 'build', 'from_zone')
+__all__ = ("GeoIDError", "parse", "build", "from_zone")
 
 
 class GeoIDError(ValueError):
-    '''Raised when an error occur while parsing or building a GeoID'''
+    """Raised when an error occur while parsing or building a GeoID"""
+
     pass
 
 
 def parse(text):
-    '''Parse a geoid from text and return a tuple (level, code, validity)'''
+    """Parse a geoid from text and return a tuple (level, code, validity)"""
     # Kept validity parsing for legacy parsing and migration.
     # Validity is parsed but ignored.
-    if '@' in text:
-        spatial, validity = text.split('@')
+    if "@" in text:
+        spatial, validity = text.split("@")
     else:
         spatial = text
-    spatial = spatial.lower().replace('/', ':')  # Backward compatibility
-    if ':' not in spatial:
-        raise GeoIDError('Bad GeoID format: {0}'.format(text))
+    spatial = spatial.lower().replace("/", ":")  # Backward compatibility
+    if ":" not in spatial:
+        raise GeoIDError("Bad GeoID format: {0}".format(text))
     # country-subset is a special case:
-    if spatial.startswith('country-subset:'):
-        level, code = spatial.split(':', 1)
+    if spatial.startswith("country-subset:"):
+        level, code = spatial.split(":", 1)
     else:
-        level, code = spatial.rsplit(':', 1)
+        level, code = spatial.rsplit(":", 1)
     return level, code
 
 
 def build(level, code):
-    '''Serialize a GeoID from its parts'''
-    return ':'.join((level, code))
+    """Serialize a GeoID from its parts"""
+    return ":".join((level, code))
 
 
 def from_zone(zone):
-    '''Build a GeoID from a given zone'''
+    """Build a GeoID from a given zone"""
     return build(zone.level, zone.code)