udata 7.0.5.dev27829__py2.py3-none-any.whl → 7.0.5.dev27885__py2.py3-none-any.whl

udata/core/dataset/rdf.py

@@ -2,17 +2,21 @@
 This module centralize dataset helpers for RDF/DCAT serialization and parsing
 '''
 import calendar
+import json
 import logging
 
 from datetime import date
 from html.parser import HTMLParser
 from dateutil.parser import parse as parse_dt
 from flask import current_app
+from geomet import wkt
 from rdflib import Graph, URIRef, Literal, BNode
 from rdflib.resource import Resource as RdfResource
 from rdflib.namespace import RDF
+from mongoengine.errors import ValidationError
 
 from udata import i18n, uris
+from udata.core.spatial.models import SpatialCoverage
 from udata.frontend.markdown import parse_html
 from udata.core.dataset.models import HarvestDatasetMetadata, HarvestResourceMetadata
 from udata.models import db, ContactPoint
@@ -334,6 +338,42 @@ def contact_point_from_rdf(rdf, dataset):
                     ContactPoint(name=name, email=email, owner=dataset.owner).save())
 
 
+def spatial_from_rdf(term):
+    if term is None:
+        return None
+
+    for object in term.objects():
+        if isinstance(object, Literal):
+            if object.datatype.__str__() == 'https://www.iana.org/assignments/media-types/application/vnd.geo+json':
+                try:
+                    geojson = json.loads(object.toPython())
+                except ValueError as e:
+                    log.warning(f"Invalid JSON in spatial GeoJSON {object.toPython()} {e}")
+                    continue
+            elif object.datatype.__str__() == 'http://www.opengis.net/rdf#wktLiteral':
+                try:
+                    # .upper() si here because geomet doesn't support Polygon but only POLYGON
+                    geojson = wkt.loads(object.toPython().strip().upper())
+                except ValueError as e:
+                    log.warning(f"Invalid JSON in spatial WKT {object.toPython()} {e}")
+                    continue
+            else:
+                continue
+
+            if geojson['type'] == 'Polygon':
+                geojson['type'] = 'MultiPolygon'
+                geojson['coordinates'] = [geojson['coordinates']]
+
+            spatial_coverage = SpatialCoverage(geom=geojson)
+
+            try:
+                spatial_coverage.clean()
+                return spatial_coverage
+            except ValidationError:
+                return None
+
+    return None
+
 def frequency_from_rdf(term):
     if isinstance(term, str):
         try:
@@ -488,7 +528,7 @@ def resource_from_rdf(graph_or_distrib, dataset=None, is_additionnal=False):
     return resource
 
 
-def dataset_from_rdf(graph, dataset=None, node=None):
+def dataset_from_rdf(graph: Graph, dataset=None, node=None):
     '''
     Create or update a dataset from a RDF/DCAT graph
     '''
@@ -509,6 +549,10 @@ def dataset_from_rdf(graph, dataset=None, node=None):
     if schema:
         dataset.schema = schema
 
+    spatial_coverage = spatial_from_rdf(d.value(DCT.spatial))
+    if spatial_coverage:
+        dataset.spatial = spatial_coverage
+
     acronym = rdf_value(d, SKOS.altLabel)
     if acronym:
         dataset.acronym = acronym

udata/forms/fields.py

@@ -180,6 +180,16 @@ class BooleanField(FieldHelper, fields.BooleanField):
         self.stacked = kwargs.pop('stacked', False)
         super(BooleanField, self).__init__(*args, **kwargs)
 
+    def process_formdata(self, valuelist):
+        # We override this so that when no value is provided
+        # the form doesn't think the value is `False` instead 
+        # the value is not present and the model can keep the
+        # existing value
+        if not valuelist:
+            return 
+
+        super().process_formdata(valuelist)
+
 
 class RadioField(FieldHelper, fields.RadioField):
     def __init__(self, *args, **kwargs):

udata/harvest/tests/dcat/bnodes.xml

@@ -5,6 +5,8 @@
   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
   xmlns:dcat="http://www.w3.org/ns/dcat#"
   xmlns:dct="http://purl.org/dc/terms/"
+  xmlns:ogc="http://www.opengis.net/ogc"
+  xmlns:geo="http://www.w3.org/2003/01/geo/wgs84_pos#"
   xmlns:dcterms="http://purl.org/dc/terms/"
   xmlns:vcard="http://www.w3.org/2006/vcard/ns#"
   xmlns:schema="http://schema.org/"
@@ -54,7 +56,13 @@
         <owl:versionInfo>1.0</owl:versionInfo>
         <dcat:distribution rdf:resource="http://data.test.org/datasets/1/resources/2"/>
         <dcat:keyword>Tag 4</dcat:keyword>
-        <dcterms:spatial rdf:resource="http://wuEurope.com/"/>
+        <dct:spatial>
+            <ogc:Polygon>
+                <geo:asWKT rdf:datatype="http://www.opengis.net/rdf#wktLiteral">
+                    wrong wkt
+                </geo:asWKT>
+            </ogc:Polygon>
+        </dct:spatial>
         <dcterms:modified rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2016-12-14T19:01:24.184120</dcterms:modified>
         <dcat:keyword>Tag 2</dcat:keyword>
         <dcat:keyword>Tag 1</dcat:keyword>
@@ -79,7 +87,13 @@
         <dcat:keyword>Tag 3</dcat:keyword>
         <dcat:distribution rdf:resource="http://data.test.org/datasets/2/resources/2"/>
         <dcterms:title>Dataset 2</dcterms:title>
-        <dcterms:spatial rdf:resource="http://wuEurope.com/"/>
+        <dct:spatial>
+            <ogc:Polygon>
+                <geo:asWKT rdf:datatype="http://www.opengis.net/rdf#wktLiteral">
+                    Polygon((4.44641288 45.54214467, 4.44641288 46.01316963, 4.75655252 46.01316963, 4.75655252 45.54214467, 4.44641288 45.54214467))
+                </geo:asWKT>
+            </ogc:Polygon>
+        </dct:spatial>
         <dcterms:identifier>2</dcterms:identifier>
         <dct:conformsTo rdf:resource="https://www.ecologie.gouv.fr/sites/default/files/R%C3%A9glementation%20IRVE.pdf" />
       </dcat:Dataset>

udata/harvest/tests/test_dcat_backend.py

@@ -156,6 +156,24 @@ class DcatBackendTest:
         assert len(datasets['1'].resources) == 2
         assert len(datasets['2'].resources) == 2
 
+
+    @pytest.mark.options(SCHEMA_CATALOG_URL='https://example.com/schemas')
+    def test_harvest_spatial(self, rmock):
+        rmock.get('https://example.com/schemas', json=ResourceSchemaMockData.get_mock_data())
+
+        filename = 'bnodes.xml'
+        url = mock_dcat(rmock, filename)
+        org = OrganizationFactory()
+        source = HarvestSourceFactory(backend='dcat', url=url, organization=org)
+
+        actions.run(source.slug)
+
+        datasets = {d.harvest.dct_identifier: d for d in Dataset.objects}
+
+        assert datasets['1'].spatial == None
+        assert datasets['2'].spatial.geom == {'type': 'MultiPolygon', 'coordinates': [[[[4.44641288, 45.54214467], [4.44641288, 46.01316963], [4.75655252, 46.01316963], [4.75655252, 45.54214467], [4.44641288, 45.54214467]]]]}
+        assert datasets['3'].spatial == None
+
     @pytest.mark.options(SCHEMA_CATALOG_URL='https://example.com/schemas')
     def test_harvest_schemas(self, rmock):
         rmock.get('https://example.com/schemas', json=ResourceSchemaMockData.get_mock_data())

udata/tests/api/test_datasets_api.py

@@ -462,6 +462,33 @@ class DatasetAPITest(APITestCase):
         dataset = Dataset.objects.first()
         self.assertEqual(len(dataset.resources), initial_length + 1)
 
+    def test_dataset_api_update_private(self):
+        user = self.login()
+        dataset = DatasetFactory(owner=user, private=True)
+        data = dataset.to_dict()
+        data['description'] = 'new description'
+        del data['private']
+
+        response = self.put(url_for('api.dataset', dataset=dataset), data)
+        self.assert200(response)
+        dataset.reload()
+        self.assertEqual(dataset.description, 'new description')
+        self.assertEqual(dataset.private, True)
+        
+        data['private'] = None
+        response = self.put(url_for('api.dataset', dataset=dataset), data)
+        self.assert200(response)
+        dataset.reload()
+        self.assertEqual(dataset.private, False)
+
+        data['private'] = True
+        response = self.put(url_for('api.dataset', dataset=dataset), data)
+        self.assert200(response)
+        dataset.reload()
+        self.assertEqual(dataset.private, True)
+
+
+
     def test_dataset_api_update_new_resource_with_extras(self):
         '''It should update a dataset with a new resource with extras'''
         user = self.login()

{udata-7.0.5.dev27829.dist-info → udata-7.0.5.dev27885.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: udata
-Version: 7.0.5.dev27829
+Version: 7.0.5.dev27885
 Summary: Open data portal
 Home-page: https://github.com/opendatateam/udata
 Author: Opendata Team
@@ -41,6 +41,7 @@ Requires-Dist: certifi ==2021.5.30
 Requires-Dist: cffi ==1.14.6
 Requires-Dist: chardet ==3.0.4
 Requires-Dist: click ==8.0.0
+Requires-Dist: colorama ==0.4.6
 Requires-Dist: cryptography ==2.8
 Requires-Dist: dnspython ==2.1.0
 Requires-Dist: email-validator ==1.1.3
@@ -63,6 +64,7 @@ Requires-Dist: flask-sitemap ==0.3.0
 Requires-Dist: flask-storage ==1.3.2
 Requires-Dist: flask-wtf ==1.0.1
 Requires-Dist: geojson ==2.5.0
+Requires-Dist: geomet ==1.1.0
 Requires-Dist: html2text ==2019.9.26
 Requires-Dist: idna ==2.10
 Requires-Dist: importlib-metadata ==6.0.0
@@ -136,6 +138,8 @@ It is collectively taken care of by members of the
 
 ## Current (in progress)
 
+- Add spatial coverage harvesting [#2959](https://github.com/opendatateam/udata/pull/2959)
+- Fix: updating a dataset without `private` do not reset `private` to `False`, the previous saved value is kept [#2955](https://github.com/opendatateam/udata/pull/2955)
 - Fix: return the correct error when no `Content-Type` is sent instead of 500 [#2967](https://github.com/opendatateam/udata/pull/2967)
 - Improve documentation for API errors [#2952](https://github.com/opendatateam/udata/pull/2965)
 

{udata-7.0.5.dev27829.dist-info → udata-7.0.5.dev27885.dist-info}/RECORD

@@ -90,7 +90,7 @@ udata/core/dataset/forms.py,sha256=auVYxLrPMdtvf2uhgEpJviHiQOSfLpBJdpZ3dXwcjNs,6
 udata/core/dataset/models.py,sha256=SK16LJP_vlSkeGrLJTFbHFNc9iHWwM6_A6DOT0SX5D4,39609
 udata/core/dataset/permissions.py,sha256=3F2J7le3_rEYNhh88o3hSRWHAAt01_yHJM6RPmvCrRo,1090
 udata/core/dataset/preview.py,sha256=puPKT3fBD7ezAcT6owh0JK1_rGNDFZOqgT223qGn3LY,2597
-udata/core/dataset/rdf.py,sha256=lVON8z1aNgMe-tyIL14HbSbr3h-CAOVDtFy8DkPh9fA,19817
+udata/core/dataset/rdf.py,sha256=7IguXzHgNcmbGY9FtiqjZ1Ju8wDxD6ahdvTj-5OzNgA,21476
 udata/core/dataset/search.py,sha256=LOHtOJYZJDPJ5S_Ch7QmmvInpw8aysmNlzbAWbMq9d0,5329
 udata/core/dataset/signals.py,sha256=TK6dfrOUitZZkGGOh6XmhYqYvIjzZpI70JTLV4k-JRM,161
 udata/core/dataset/tasks.py,sha256=VB1sQ6Fwbax46IRLGyZUDPGgGOWBYrzAlKzV3npDCyM,8412
@@ -243,7 +243,7 @@ udata/features/transfer/models.py,sha256=YjX0O53DQS8jUU9pb6hFCsBx-eo8g8DOAD7EHZM
 udata/features/transfer/notifications.py,sha256=oh_uP0XlEEeT7747A292WkjAjAyO4d0YK28oBICN24Y,1003
 udata/features/transfer/permissions.py,sha256=-iEgQkIfLYDGarhDVp3TMas4ABl_Pc4KJTSP8pcewws,953
 udata/forms/__init__.py,sha256=ZxakXdJMwsM_YFT96Z2X24J5YpbpnP_VZ_VEh15gK3w,820
-udata/forms/fields.py,sha256=rR32Jw2OO5yCvBJ42iNhKjrccleNGg-Iz5cCokmfgHE,28783
+udata/forms/fields.py,sha256=WTCAc4KVPS6EanpUPuan_reDM9gwx4K2wdvJBxu_pTY,29129
 udata/forms/validators.py,sha256=EqCJloDOR3BonrR34N_ZaMGBXr0iuliDc82khc98e20,2816
 udata/forms/widgets.py,sha256=P-D3uYfNH5OhAa-SqNoXROg_DuDlnpv_DaO_e4LcJ5I,1380
 udata/frontend/__init__.py,sha256=Xc71TI6a8b_QR_wqEOc1tty2y1CwHVbV4tS04nElme0,3494
@@ -269,7 +269,7 @@ udata/harvest/tests/factories.py,sha256=CbQORC1OJ1_Agtv_3LjCXysNumjMYlROwZPSEAHo
 udata/harvest/tests/test_actions.py,sha256=7xSpouCAcf5p_bd38zHCyPN7sKWUUZXA7IlpI-yNVrQ,27603
 udata/harvest/tests/test_api.py,sha256=QXhseHfnkBEmMbIJzroMdDYGLDj6Njal1s-2sn0xhEM,14888
 udata/harvest/tests/test_base_backend.py,sha256=JA8Df1Eu-lEPLZfxyK81bsmT6exOjV_3PtKHJekAp5g,12092
-udata/harvest/tests/test_dcat_backend.py,sha256=3n_nI2Of8M1kq4nD9oGgWqSeOIaxQXBrMqi6e8lBlCQ,23930
+udata/harvest/tests/test_dcat_backend.py,sha256=8bdpPMh9WsnUDpO19ZoMHfUnU1_fe8Zl_hLSntFwXwI,24761
 udata/harvest/tests/test_filters.py,sha256=V2HFZlexIJa6r1DX6g2ktvIgjg4gSY11QPfPOd3_Oug,2370
 udata/harvest/tests/test_models.py,sha256=p2VazyrPXSArBuf8Kf19TGPcQ86SnOGCGmvjcMOw0s0,924
 udata/harvest/tests/test_notifications.py,sha256=ZwtwioittW3XcZc0x6zbHjs1dVaAxPytlVymnJa5w0E,817
@@ -278,7 +278,7 @@ udata/harvest/tests/csw_dcat/geonetworkv4-page-1.xml,sha256=k2pKidlQvJpoltGFm9HN
 udata/harvest/tests/csw_dcat/geonetworkv4-page-3.xml,sha256=fsN0E4TVd_ts-sYA612yBP-gRAwpyQWqJdNm7ohczbs,20945
 udata/harvest/tests/csw_dcat/geonetworkv4-page-5.xml,sha256=0VmPp1kspik7YAmOFyr-3yJLzWGA6kuQp_x_w-W385o,21213
 udata/harvest/tests/dcat/bnodes.jsonld,sha256=Leqny-ccp30564yojQYYckw_HKbhR0f5qUCaavc2ruE,7964
-udata/harvest/tests/dcat/bnodes.xml,sha256=nltHV8RtvitU6OiwX6CjGtaKNlvzJZY0IAO3CmOFmak,8562
+udata/harvest/tests/dcat/bnodes.xml,sha256=KHrVtHiHrWLIT7vxSK7xpYlClCWSWvczeOp-QzzBZVc,9128
 udata/harvest/tests/dcat/catalog.xml,sha256=pBVSNZzA4gkgxHCttrjuBBVSumLFgXgvtSgnHr0ybk4,9239
 udata/harvest/tests/dcat/flat.jsonld,sha256=BAw08MDhtW9Px3q6RAoTIqO_OwJmAwBS9EpC8BY_x98,8459
 udata/harvest/tests/dcat/geonetwork.xml,sha256=9_pksE74Zzkbgs9okj6hEbo8CJS0FZjEnIdvopKfm7k,7928
@@ -557,7 +557,7 @@ udata/tests/api/__init__.py,sha256=Tz_WigHLDlnJNKOKzEAnJswkKiLtHlIpCE54-wgocgM,9
 udata/tests/api/test_auth_api.py,sha256=3Zhn2A29poZIcCJ_R9_-LkR3xOFUTw1aTquiZVXQ2F0,20306
 udata/tests/api/test_base_api.py,sha256=DRX5nuFIj51GFmMIAxUzoW1yiq1apNgr1vS4U4agzeg,2319
 udata/tests/api/test_contact_points.py,sha256=MJm8B06iaUqIZCqxll3NViFwUCxwqZZ4u9e9s1h8MgU,1056
-udata/tests/api/test_datasets_api.py,sha256=i52H9Tg8qKBk2sS9I6v85Zbz3N8Wnx0Wu4Uadz8LjW8,80444
+udata/tests/api/test_datasets_api.py,sha256=kCl55lyWgDxCwkYTa1HVMNlR3GBJiAcwOCkvh1BQzXw,81372
 udata/tests/api/test_fields.py,sha256=OW85Z5MES5HeWOpapeem8OvR1cIcrqW-xMWpdZO4LZ8,1033
 udata/tests/api/test_follow_api.py,sha256=0h54P_Dfbo07u6tg0Rbai1WWgWb19ZLN2HGv4oLCWfg,3383
 udata/tests/api/test_me_api.py,sha256=-JskdyxcbREceC1PxMj9NUs6QBE3YzKa5TXhHIaPy-o,14275
@@ -649,9 +649,9 @@ udata/translations/pt/LC_MESSAGES/udata.mo,sha256=dXTr2zzGPgclSMzAJo4qhOEVHYWYP7
 udata/translations/pt/LC_MESSAGES/udata.po,sha256=G-MNwE4Gih-4w-xNAjp7PAvQOeW1uyM-27Z6O08_SCc,42104
 udata/translations/sr/LC_MESSAGES/udata.mo,sha256=5bvKeJp9nqk76tZsgbGC2qnbpJws5lODdMcezr6LKbQ,28553
 udata/translations/sr/LC_MESSAGES/udata.po,sha256=d_uBu93oIsmeewv8QzrB2Z8tmkUXn8juwT5OiySALog,48684
-udata-7.0.5.dev27829.dist-info/LICENSE,sha256=V8j_M8nAz8PvAOZQocyRDX7keai8UJ9skgmnwqETmdY,34520
-udata-7.0.5.dev27829.dist-info/METADATA,sha256=H3hZ8-KhDGt4cc3IS2hncX4WaOQ_JeLgIlJxo5zq9LE,117925
-udata-7.0.5.dev27829.dist-info/WHEEL,sha256=-G_t0oGuE7UD0DrSpVZnq1hHMBV9DD2XkS5v7XpmTnk,110
-udata-7.0.5.dev27829.dist-info/entry_points.txt,sha256=ZqIUHhOth0MMQvMIeuhODbUCDwjR-Hvo7PaKrMwTKuQ,384
-udata-7.0.5.dev27829.dist-info/top_level.txt,sha256=39OCg-VWFWOq4gCKnjKNu-s3OwFlZIu_dVH8Gl6ndHw,12
-udata-7.0.5.dev27829.dist-info/RECORD,,
+udata-7.0.5.dev27885.dist-info/LICENSE,sha256=V8j_M8nAz8PvAOZQocyRDX7keai8UJ9skgmnwqETmdY,34520
+udata-7.0.5.dev27885.dist-info/METADATA,sha256=kcy_O4ue_mJRqxGZ2UdLKoxeigd80IF9igrM5DCOfxI,118247
+udata-7.0.5.dev27885.dist-info/WHEEL,sha256=-G_t0oGuE7UD0DrSpVZnq1hHMBV9DD2XkS5v7XpmTnk,110
+udata-7.0.5.dev27885.dist-info/entry_points.txt,sha256=ZqIUHhOth0MMQvMIeuhODbUCDwjR-Hvo7PaKrMwTKuQ,384
+udata-7.0.5.dev27885.dist-info/top_level.txt,sha256=39OCg-VWFWOq4gCKnjKNu-s3OwFlZIu_dVH8Gl6ndHw,12
+udata-7.0.5.dev27885.dist-info/RECORD,,