udata 7.0.6__py2.py3-none-any.whl → 7.0.6.dev28180__py2.py3-none-any.whl

udata/harvest/tests/test_dcat_backend.py

@@ -1,22 +1,21 @@
-from datetime import date
 import logging
 import os
-import re
+from flask import current_app
 
 import pytest
 
+from datetime import date
 import boto3
-from flask import current_app
 import xml.etree.ElementTree as ET
-
 from udata.harvest.models import HarvestJob
+
 from udata.models import Dataset
 from udata.core.organization.factories import OrganizationFactory
 from udata.core.dataset.factories import LicenseFactory, ResourceSchemaMockData
 from udata.storage.s3 import get_from_json
 
 from .factories import HarvestSourceFactory
-from ..backends.dcat import URIS_TO_REPLACE, CswIso19139DcatBackend
+from ..backends.dcat import URIS_TO_REPLACE
 from .. import actions
 
 log = logging.getLogger(__name__)
@@ -161,21 +160,6 @@ class DcatBackendTest:
         assert len(datasets['1'].resources) == 2
         assert len(datasets['2'].resources) == 2
 
-    def test_harvest_literal_spatial(self, rmock):
-        url = mock_dcat(rmock, 'evian.json')
-        org = OrganizationFactory()
-        source = HarvestSourceFactory(backend='dcat',
-                                      url=url,
-                                      organization=org)
-        
-        actions.run(source.slug)
-
-        datasets = {d.harvest.dct_identifier: d for d in Dataset.objects}
-        assert len(datasets) == 8
-        assert datasets['https://www.arcgis.com/home/item.html?id=f6565516d1354383b25793e630cf3f2b&sublayer=5'].spatial is not None
-        assert datasets['https://www.arcgis.com/home/item.html?id=f6565516d1354383b25793e630cf3f2b&sublayer=5'].spatial.geom == {'type': 'MultiPolygon', 'coordinates': [[[[6.5735, 46.3912], [6.6069, 46.3912], [6.6069, 46.4028], [6.5735, 46.4028], [6.5735, 46.3912]]]]}
-
-
     @pytest.mark.skip(reason="Mocking S3 requires `moto` which is not available for our current Python 3.7. We can manually test it.")
     @pytest.mark.options(SCHEMA_CATALOG_URL='https://example.com/schemas', HARVEST_JOBS_RETENTION_DAYS=0)
     # @mock_s3
@@ -211,8 +195,8 @@ class DcatBackendTest:
         assert resources_by_title['Resource 1-2'].schema.name == None
         assert resources_by_title['Resource 1-2'].schema.version == None
 
-        assert datasets['2'].schema.name == 'RGF93 / Lambert-93 (EPSG:2154)'
-        assert datasets['2'].schema.url == 'http://inspire.ec.europa.eu/glossary/SpatialReferenceSystem'
+        assert datasets['2'].schema.name == None
+        assert datasets['2'].schema.url == 'https://www.ecologie.gouv.fr/sites/default/files/R%C3%A9glementation%20IRVE.pdf'
         resources_by_title = { resource['title']: resource for resource in datasets['2'].resources }
 
         # Unknown schema are kept as they were provided
@@ -281,8 +265,8 @@ class DcatBackendTest:
         assert resources_by_title['Resource 1-2'].schema.name == None
         assert resources_by_title['Resource 1-2'].schema.version == None
 
-        assert datasets['2'].schema.name == 'RGF93 / Lambert-93 (EPSG:2154)'
-        assert datasets['2'].schema.url == 'http://inspire.ec.europa.eu/glossary/SpatialReferenceSystem'
+        assert datasets['2'].schema.name == None
+        assert datasets['2'].schema.url == 'https://www.ecologie.gouv.fr/sites/default/files/R%C3%A9glementation%20IRVE.pdf'
         resources_by_title = { resource['title']: resource for resource in datasets['2'].resources }
 
         # Unknown schema are kept as they were provided
@@ -630,7 +614,7 @@ class DcatBackendTest:
 
 
 @pytest.mark.usefixtures('clean_db')
-@pytest.mark.options(PLUGINS=['csw'])
+@pytest.mark.options(PLUGINS=['csw-dcat'])
 class CswDcatBackendTest:
 
     def test_geonetworkv4(self, rmock):
@@ -678,60 +662,3 @@ class CswDcatBackendTest:
 
         assert 'User-Agent' in get_mock.last_request.headers
         assert get_mock.last_request.headers['User-Agent'] == 'uData/0.1 csw-dcat'
-
-
-@pytest.mark.usefixtures('clean_db')
-@pytest.mark.options(PLUGINS=['csw'])
-class CswIso19139DcatBackendTest:
-
-    def test_geo2france(self, rmock):
-
-        with open(os.path.join(CSW_DCAT_FILES_DIR, "XSLT.xml"), "r") as f:
-            xslt = f.read()
-        url = mock_csw_pagination(rmock, 'geonetwork/srv/eng/csw.rdf', 'geonetwork-iso-page-{}.xml')
-        rmock.get(CswIso19139DcatBackend.XSL_URL, text=xslt)
-        org = OrganizationFactory()
-        source = HarvestSourceFactory(backend='csw-iso-19139',
-                                      url=url,
-                                      organization=org)
-
-        actions.run(source.slug)
-
-        source.reload()
-
-        job = source.get_last_job()
-        assert len(job.items) == 6
-
-        datasets = {d.harvest.dct_identifier: d for d in Dataset.objects}
-
-        assert len(datasets) == 6
-
-        # First dataset
-        # dataset identifier is gmd:RS_Identifier > gmd:codeSpace + gmd:code
-        dataset = datasets['http://catalogue.geo-ide.developpement-durable.gouv.fr/fr-120066022-orphan-residentifier-140d31c6-643d-42a9-85df-2737a118e144']
-        assert dataset.title == "Plan local d'urbanisme de la commune de Cartigny"
-        assert dataset.description == "Le présent standard de données COVADIS concerne les documents de plans locaux d'urbanisme (PLU) et les plans d'occupation des sols (POS qui valent PLU)."
-        assert set(dataset.tags) == set([
-            'amenagement-urbanisme-zonages-planification', 'cartigny',
-            'document-durbanisme', 'donnees-ouvertes', 'plu', 'usage-des-sols'
-        ])
-        assert dataset.harvest.created_at.date() == date(2017, 10, 7)
-        assert dataset.spatial.geom == {'type': 'MultiPolygon', 'coordinates':
-            [[[[3.28133559, 50.48188019], [1.31279111, 50.48188019], [1.31279111, 49.38547516], [3.28133559, 49.38547516], [3.28133559, 50.48188019]]]]
-        }
-        assert dataset.contact_point.name == 'DDTM 80 (Direction Départementale des Territoires et de la Mer de la Somme)'
-        assert dataset.contact_point.email == 'ddtm-sap-bsig@somme.gouv.fr'
-
-        # License is not properly mapped in XSLT conversion
-        assert dataset.license is None
-
-        # Distributions don't get properly mapped to distribution with this XSLT if missing CI_OnLineFunctionCode.
-        # A CI_OnLineFunctionCode was added explicitely on one of the Online Resources.
-        # (See mapping at: https://semiceu.github.io/GeoDCAT-AP/releases/2.0.0/#resource-locator---on-line-resource)
-        assert len(dataset.resources) == 1
-        resource = dataset.resources[0]
-        assert resource.title == 'Téléchargement direct du lot et des documents associés'
-        assert resource.url == 'http://atom.geo-ide.developpement-durable.gouv.fr/atomArchive/GetResource?id=fr-120066022-ldd-cab63273-b3ae-4e8a-ae1c-6192e45faa94&datasetAggregate=true'
-        
-        # Sadly resource format is parsed as a blank node. Format parsing should be improved.
-        assert re.match(r'n[0-9a-f]{32}', resource.format)

udata/rdf.py

@@ -12,7 +12,6 @@ from rdflib.namespace import (
     Namespace, NamespaceManager, DCTERMS, SKOS, FOAF, XSD, RDFS, RDF
 )
 from rdflib.util import SUFFIX_FORMAT_MAP, guess_format as raw_guess_format
-from udata import uris
 from udata.models import FieldValidationError, Schema, ResourceSchema
 from mongoengine import ValidationError
 
@@ -76,7 +75,6 @@ ACCEPTED_MIME_TYPES = {
     'application/ld+json': 'json-ld',
     'application/json': 'json-ld',
     'application/trig': 'trig',
-    'text/xml': 'xml',
     # Available but not activated
     # 'application/n-quads': 'nquads',
     # 'text/xml': 'trix',
@@ -238,26 +236,7 @@ def schema_from_rdf(rdf):
     if isinstance(resource, (URIRef, Literal)):
         schema.url = resource.toPython()
     elif isinstance(resource, RdfResource):
-        # We try to get the schema "correct" URL.
-        # 1. The identifier of the DCT.conformsTo
-        # 2. The DCT.type inside the DCT.conformsTo (from some example it's the most precise one)
-        # (other not currently used RDF.type)
-        url = None
-        try:
-            url = uris.validate(resource.identifier.toPython())
-        except uris.ValidationError:
-            try:
-                type = resource.value(DCT.type)
-                if type is not None:
-                    url = uris.validate(type.identifier.toPython())
-            except uris.ValidationError:
-                pass
-            pass
-
-        if url is None:
-            return None
-
-        schema.url = url
+        schema.url = resource.identifier.toPython()
         schema.name = resource.value(DCT.title)
     else:
         return None

udata/search/__init__.py

@@ -24,7 +24,7 @@ def reindex(classname, id):
     document = adapter_class.serialize(obj)
     if adapter_class.is_indexable(obj):
         log.info('Indexing %s (%s)', model.__name__, obj.id)
-        url = f"{current_app.config['SEARCH_SERVICE_API_URL']}{adapter_class.search_url}index"
+        url = f"{current_app.config['SEARCH_SERVICE_API_URL']}{adapter_class.search_url}/index"
         try:
             payload = {
                 'document': document
@@ -35,7 +35,7 @@ def reindex(classname, id):
             log.exception('Unable to index/unindex %s "%s"', model.__name__, str(obj.id))
     else:
         log.info('Unindexing %s (%s)', model.__name__, obj.id)
-        url = f"{current_app.config['SEARCH_SERVICE_API_URL']}{adapter_class.search_url}{str(obj.id)}/unindex"
+        url = f"{current_app.config['SEARCH_SERVICE_API_URL']}{adapter_class.search_url}/{str(obj.id)}/unindex"
         try:
             r = requests.delete(url)
             if r.status_code == 404: