udata 7.0.5.dev27838__py2.py3-none-any.whl → 7.0.5.dev27942__py2.py3-none-any.whl

udata/core/dataset/rdf.py

@@ -2,17 +2,21 @@
 This module centralize dataset helpers for RDF/DCAT serialization and parsing
 '''
 import calendar
+import json
 import logging
 
 from datetime import date
 from html.parser import HTMLParser
 from dateutil.parser import parse as parse_dt
 from flask import current_app
+from geomet import wkt
 from rdflib import Graph, URIRef, Literal, BNode
 from rdflib.resource import Resource as RdfResource
 from rdflib.namespace import RDF
+from mongoengine.errors import ValidationError
 
 from udata import i18n, uris
+from udata.core.spatial.models import SpatialCoverage
 from udata.frontend.markdown import parse_html
 from udata.core.dataset.models import HarvestDatasetMetadata, HarvestResourceMetadata
 from udata.models import db, ContactPoint
@@ -334,6 +338,42 @@ def contact_point_from_rdf(rdf, dataset):
                     ContactPoint(name=name, email=email, owner=dataset.owner).save())
 
 
+def spatial_from_rdf(term):
+    if term is None:
+        return None
+
+    for object in term.objects():
+        if isinstance(object, Literal):
+            if object.datatype.__str__() == 'https://www.iana.org/assignments/media-types/application/vnd.geo+json':
+                try:
+                    geojson = json.loads(object.toPython())
+                except ValueError as e:
+                    log.warning(f"Invalid JSON in spatial GeoJSON {object.toPython()} {e}")
+                    continue
+            elif object.datatype.__str__() == 'http://www.opengis.net/rdf#wktLiteral':
+                try:
+                    # .upper() si here because geomet doesn't support Polygon but only POLYGON
+                    geojson = wkt.loads(object.toPython().strip().upper())
+                except ValueError as e:
+                    log.warning(f"Invalid JSON in spatial WKT {object.toPython()} {e}")
+                    continue
+            else:
+                continue
+
+            if geojson['type'] == 'Polygon':
+                geojson['type'] = 'MultiPolygon'
+                geojson['coordinates'] = [geojson['coordinates']]
+
+            spatial_coverage = SpatialCoverage(geom=geojson)
+
+            try:
+                spatial_coverage.clean()
+                return spatial_coverage
+            except ValidationError:
+                return None
+
+    return None
+
 def frequency_from_rdf(term):
     if isinstance(term, str):
         try:
@@ -488,7 +528,7 @@ def resource_from_rdf(graph_or_distrib, dataset=None, is_additionnal=False):
     return resource
 
 
-def dataset_from_rdf(graph, dataset=None, node=None):
+def dataset_from_rdf(graph: Graph, dataset=None, node=None):
     '''
     Create or update a dataset from a RDF/DCAT graph
     '''
@@ -509,6 +549,10 @@ def dataset_from_rdf(graph, dataset=None, node=None):
     if schema:
         dataset.schema = schema
 
+    spatial_coverage = spatial_from_rdf(d.value(DCT.spatial))
+    if spatial_coverage:
+        dataset.spatial = spatial_coverage
+
     acronym = rdf_value(d, SKOS.altLabel)
     if acronym:
         dataset.acronym = acronym

udata/core/organization/csv.py

@@ -1,3 +1,4 @@
+from udata.core.dataset.models import Dataset
 from udata.frontend import csv
 
 from .models import Organization
@@ -5,6 +6,8 @@ from .models import Organization
 
 @csv.adapter(Organization)
 class OrganizationCsvAdapter(csv.Adapter):
+    downloads_counts = None
+
     fields = (
         'id',
         'name',
@@ -18,4 +21,27 @@ class OrganizationCsvAdapter(csv.Adapter):
     )
 
     def dynamic_fields(self):
-        return csv.metric_fields(Organization)
+        return csv.metric_fields(Organization) + self.get_dynamic_field_downloads()
+    
+    def get_dynamic_field_downloads(self):
+        downloads_counts = self.get_downloads_counts()
+        return [('downloads', lambda o: downloads_counts.get(str(o.id), 0))]
+    
+    def get_downloads_counts(self):
+        '''
+        Prefetch all the resources' downloads for all selected organization into memory
+        '''
+        if self.downloads_counts is not None:
+            return self.downloads_counts
+
+        self.downloads_counts = {}
+
+        ids = [o.id for o in self.queryset]
+        for dataset in Dataset.objects(organization__in=ids):
+            org_id = str(dataset.organization.id)
+            if self.downloads_counts.get(org_id) is None:
+                self.downloads_counts[org_id] = 0
+
+            self.downloads_counts[org_id] += sum(resource.metrics.get('views', 0) for resource in dataset.resources)
+
+        return self.downloads_counts

udata/frontend/csv.py

@@ -58,8 +58,8 @@ class Adapter(object):
                     else:
                         field_tuple = (name, self.getter(*field))
                 except Exception as e:  # Catch all errors intentionally.
-                    log.error('Error exporting CSV for {name}: {error}'.format(
-                        name=self.__class__.__name__, error=e))
+                    log.error('Error exporting CSV for {name}: {error_class} {error}'.format(
+                        name=self.__class__.__name__, error_class=e.__class__.__name__, error=e))
                 self._fields.append(field_tuple)
         return self._fields
 
@@ -89,8 +89,8 @@ class Adapter(object):
                 try:
                     content = safestr(getter(obj))
                 except Exception as e:  # Catch all errors intentionally.
-                    log.error('Error exporting CSV for {name}: {error}'.format(
-                        name=self.__class__.__name__, error=e))
+                    log.error('Error exporting CSV for {name}: {error_class} {error}'.format(
+                        name=self.__class__.__name__, error_class=e.__class__.__name__, error=e))
             row.append(content)
         return row
 
@@ -130,8 +130,8 @@ class NestedAdapter(Adapter):
                     else:
                         field_tuple = (name, self.getter(*field))
                 except Exception as e:  # Catch all errors intentionally.
-                    log.error('Error exporting CSV for {name}: {error}'.format(
-                        name=self.__class__.__name__, error=e))
+                    log.error('Error exporting CSV for {name}: {error_class} {error}'.format(
+                        name=self.__class__.__name__, error_class=e.__class__.__name__, error=e))
                 self._nested_fields.append(field_tuple)
         return self._nested_fields
 
@@ -155,8 +155,8 @@ class NestedAdapter(Adapter):
                 try:
                     content = safestr(getter(nested))
                 except Exception as e:  # Catch all errors intentionally.
-                    log.error('Error exporting CSV for {name}: {error}'.format(
-                        name=self.__class__.__name__, error=e))
+                    log.error('Error exporting CSV for {name}: {error_class} {error}'.format(
+                        name=self.__class__.__name__, error_class=e.__class__.__name__, error=e))
             row.append(content)
         return row
 

udata/harvest/tests/dcat/bnodes.xml

@@ -5,6 +5,8 @@
   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
   xmlns:dcat="http://www.w3.org/ns/dcat#"
   xmlns:dct="http://purl.org/dc/terms/"
+  xmlns:ogc="http://www.opengis.net/ogc"
+  xmlns:geo="http://www.w3.org/2003/01/geo/wgs84_pos#"
   xmlns:dcterms="http://purl.org/dc/terms/"
   xmlns:vcard="http://www.w3.org/2006/vcard/ns#"
   xmlns:schema="http://schema.org/"
@@ -54,7 +56,13 @@
         <owl:versionInfo>1.0</owl:versionInfo>
         <dcat:distribution rdf:resource="http://data.test.org/datasets/1/resources/2"/>
         <dcat:keyword>Tag 4</dcat:keyword>
-        <dcterms:spatial rdf:resource="http://wuEurope.com/"/>
+        <dct:spatial>
+            <ogc:Polygon>
+                <geo:asWKT rdf:datatype="http://www.opengis.net/rdf#wktLiteral">
+                    wrong wkt
+                </geo:asWKT>
+            </ogc:Polygon>
+        </dct:spatial>
         <dcterms:modified rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2016-12-14T19:01:24.184120</dcterms:modified>
         <dcat:keyword>Tag 2</dcat:keyword>
         <dcat:keyword>Tag 1</dcat:keyword>
@@ -79,7 +87,13 @@
         <dcat:keyword>Tag 3</dcat:keyword>
         <dcat:distribution rdf:resource="http://data.test.org/datasets/2/resources/2"/>
         <dcterms:title>Dataset 2</dcterms:title>
-        <dcterms:spatial rdf:resource="http://wuEurope.com/"/>
+        <dct:spatial>
+            <ogc:Polygon>
+                <geo:asWKT rdf:datatype="http://www.opengis.net/rdf#wktLiteral">
+                    Polygon((4.44641288 45.54214467, 4.44641288 46.01316963, 4.75655252 46.01316963, 4.75655252 45.54214467, 4.44641288 45.54214467))
+                </geo:asWKT>
+            </ogc:Polygon>
+        </dct:spatial>
         <dcterms:identifier>2</dcterms:identifier>
         <dct:conformsTo rdf:resource="https://www.ecologie.gouv.fr/sites/default/files/R%C3%A9glementation%20IRVE.pdf" />
       </dcat:Dataset>

udata/harvest/tests/test_dcat_backend.py

@@ -156,6 +156,24 @@ class DcatBackendTest:
         assert len(datasets['1'].resources) == 2
         assert len(datasets['2'].resources) == 2
 
+
+    @pytest.mark.options(SCHEMA_CATALOG_URL='https://example.com/schemas')
+    def test_harvest_spatial(self, rmock):
+        rmock.get('https://example.com/schemas', json=ResourceSchemaMockData.get_mock_data())
+
+        filename = 'bnodes.xml'
+        url = mock_dcat(rmock, filename)
+        org = OrganizationFactory()
+        source = HarvestSourceFactory(backend='dcat', url=url, organization=org)
+
+        actions.run(source.slug)
+
+        datasets = {d.harvest.dct_identifier: d for d in Dataset.objects}
+
+        assert datasets['1'].spatial == None
+        assert datasets['2'].spatial.geom == {'type': 'MultiPolygon', 'coordinates': [[[[4.44641288, 45.54214467], [4.44641288, 46.01316963], [4.75655252, 46.01316963], [4.75655252, 45.54214467], [4.44641288, 45.54214467]]]]}
+        assert datasets['3'].spatial == None
+
     @pytest.mark.options(SCHEMA_CATALOG_URL='https://example.com/schemas')
     def test_harvest_schemas(self, rmock):
         rmock.get('https://example.com/schemas', json=ResourceSchemaMockData.get_mock_data())