udata 7.0.5.dev28172__py2.py3-none-any.whl → 7.0.6__py2.py3-none-any.whl

udata/harvest/tests/dcat/bnodes.xml

@@ -95,7 +95,7 @@
             </ogc:Polygon>
         </dct:spatial>
         <dcterms:identifier>2</dcterms:identifier>
-        <dct:conformsTo rdf:resource="https://www.ecologie.gouv.fr/sites/default/files/R%C3%A9glementation%20IRVE.pdf" />
+        <dct:conformsTo rdf:nodeID="Ne0189e93917c4f67a412fc44883322e7"/>
       </dcat:Dataset>
     </dcat:dataset>
     <dcterms:modified rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2016-12-15T09:19:51.723691</dcterms:modified>
@@ -154,4 +154,10 @@
   <foaf:Organization rdf:about="http://data.test.org/organizations/1">
     <foaf:name>An Organization</foaf:name>
   </foaf:Organization>
+  <rdf:Description rdf:nodeID="Ne0189e93917c4f67a412fc44883322e7">
+    <rdf:type rdf:resource="http://purl.org/dc/terms/Standard"/>
+    <rdf:type rdf:resource="http://www.w3.org/2004/02/skos/core#Concept"/>
+    <dct:type rdf:resource="http://inspire.ec.europa.eu/glossary/SpatialReferenceSystem"/>
+    <dct:title xml:lang="fr">RGF93 / Lambert-93 (EPSG:2154)</dct:title>
+  </rdf:Description>
 </rdf:RDF>

udata/harvest/tests/dcat/evian.json

@@ -0,0 +1,464 @@
+{
+	"@context": "https://project-open-data.cio.gov/v1.1/schema/catalog.jsonld",
+	"@type": "dcat:Catalog",
+	"conformsTo": "https://project-open-data.cio.gov/v1.1/schema",
+	"describedBy": "https://project-open-data.cio.gov/v1.1/schema/catalog.json",
+	"dataset": [
+		{
+			"@type": "dcat:Dataset",
+			"identifier": "https://www.arcgis.com/home/item.html?id=f6565516d1354383b25793e630cf3f2b&sublayer=5",
+			"landingPage": "https://ville-d-evian-opendata-ville-evian.hub.arcgis.com/maps/f6565516d1354383b25793e630cf3f2b_5",
+			"title": "stationnement velos",
+			"description": "<DIV STYLE=\"text-align:Left;\"><DIV><DIV><P><SPAN>Emplacement et description des stationnement vélos à Evian-les-Bains</SPAN></P></DIV></DIV></DIV>",
+			"keyword": [
+				"mobilite"
+			],
+			"issued": "2022-07-29T10:07:13.000Z",
+			"modified": "2022-07-29T11:29:07.556Z",
+			"publisher": {
+				"name": "Ville d'Evian-les-Bains"
+			},
+			"contactPoint": {
+				"@type": "vcard:Contact",
+				"fn": "sigevian"
+			},
+			"accessLevel": "public",
+			"spatial": "6.5735,46.3912,6.6069,46.4028",
+			"license": "<DIV STYLE=\"text-align:Left;\"><DIV><DIV><P><SPAN>Soumis à la licence Etalab : https://www.etalab.gouv.fr/licence-ouverte-open-licence/</SPAN></P></DIV></DIV></DIV>",
+			"distribution": [
+				{
+					"@type": "dcat:Distribution",
+					"title": "ArcGIS Hub Dataset",
+					"format": "Web Page",
+					"mediaType": "text/html",
+					"accessURL": "https://ville-d-evian-opendata-ville-evian.hub.arcgis.com/maps/f6565516d1354383b25793e630cf3f2b_5"
+				},
+				{
+					"@type": "dcat:Distribution",
+					"title": "ArcGIS GeoService",
+					"format": "ArcGIS GeoServices REST API",
+					"mediaType": "application/json",
+					"accessURL": "https://services.arcgis.com/Hbks8qlTugfLqQka/arcgis/rest/services/stationnement_velo/FeatureServer/5"
+				},
+				{
+					"@type": "dcat:Distribution",
+					"title": "CSV",
+					"format": "CSV",
+					"mediaType": "text/csv",
+					"accessURL": "https://ville-d-evian-opendata-ville-evian.hub.arcgis.com/api/download/v1/items/f6565516d1354383b25793e630cf3f2b/csv?layers=5"
+				},
+				{
+					"@type": "dcat:Distribution",
+					"title": "GeoJSON",
+					"format": "GeoJSON",
+					"mediaType": "application/vnd.geo+json",
+					"accessURL": "https://ville-d-evian-opendata-ville-evian.hub.arcgis.com/api/download/v1/items/f6565516d1354383b25793e630cf3f2b/geojson?layers=5"
+				},
+				{
+					"@type": "dcat:Distribution",
+					"title": "Shapefile",
+					"format": "ZIP",
+					"mediaType": "application/zip",
+					"accessURL": "https://ville-d-evian-opendata-ville-evian.hub.arcgis.com/api/download/v1/items/f6565516d1354383b25793e630cf3f2b/shapefile?layers=5"
+				},
+				{
+					"@type": "dcat:Distribution",
+					"title": "KML",
+					"format": "KML",
+					"mediaType": "application/vnd.google-earth.kml+xml",
+					"accessURL": "https://ville-d-evian-opendata-ville-evian.hub.arcgis.com/api/download/v1/items/f6565516d1354383b25793e630cf3f2b/kml?layers=5"
+				}
+			],
+			"theme": [
+				"geospatial"
+			]
+		},
+		{
+			"@type": "dcat:Dataset",
+			"identifier": "https://www.arcgis.com/home/item.html?id=cb6aa94ccb21492396e799d1ad282c02&sublayer=8",
+			"landingPage": "https://ville-d-evian-opendata-ville-evian.hub.arcgis.com/maps/cb6aa94ccb21492396e799d1ad282c02_8",
+			"title": "poubelles",
+			"description": "<DIV STYLE=\"text-align:Left;\"><DIV><DIV><P><SPAN>Emplacement et description des poubelles à Evian-les-Bains</SPAN></P></DIV></DIV></DIV>",
+			"keyword": [
+				"environnement"
+			],
+			"issued": "2022-07-29T09:08:13.000Z",
+			"modified": "2022-07-29T11:33:36.369Z",
+			"publisher": {
+				"name": "Ville d'Evian-les-Bains"
+			},
+			"contactPoint": {
+				"@type": "vcard:Contact",
+				"fn": "sigevian"
+			},
+			"accessLevel": "public",
+			"spatial": "6.5658,46.3835,6.6141,46.4047",
+			"license": "<DIV STYLE=\"text-align:Left;\"><DIV><DIV><P><SPAN>Soumis à la licence Etalab : https://www.etalab.gouv.fr/licence-ouverte-open-licence/</SPAN></P></DIV></DIV></DIV>",
+			"distribution": [
+				{
+					"@type": "dcat:Distribution",
+					"title": "ArcGIS Hub Dataset",
+					"format": "Web Page",
+					"mediaType": "text/html",
+					"accessURL": "https://ville-d-evian-opendata-ville-evian.hub.arcgis.com/maps/cb6aa94ccb21492396e799d1ad282c02_8"
+				},
+				{
+					"@type": "dcat:Distribution",
+					"title": "ArcGIS GeoService",
+					"format": "ArcGIS GeoServices REST API",
+					"mediaType": "application/json",
+					"accessURL": "https://services.arcgis.com/Hbks8qlTugfLqQka/arcgis/rest/services/poubelles/FeatureServer/8"
+				},
+				{
+					"@type": "dcat:Distribution",
+					"title": "CSV",
+					"format": "CSV",
+					"mediaType": "text/csv",
+					"accessURL": "https://ville-d-evian-opendata-ville-evian.hub.arcgis.com/api/download/v1/items/cb6aa94ccb21492396e799d1ad282c02/csv?layers=8"
+				},
+				{
+					"@type": "dcat:Distribution",
+					"title": "GeoJSON",
+					"format": "GeoJSON",
+					"mediaType": "application/vnd.geo+json",
+					"accessURL": "https://ville-d-evian-opendata-ville-evian.hub.arcgis.com/api/download/v1/items/cb6aa94ccb21492396e799d1ad282c02/geojson?layers=8"
+				},
+				{
+					"@type": "dcat:Distribution",
+					"title": "Shapefile",
+					"format": "ZIP",
+					"mediaType": "application/zip",
+					"accessURL": "https://ville-d-evian-opendata-ville-evian.hub.arcgis.com/api/download/v1/items/cb6aa94ccb21492396e799d1ad282c02/shapefile?layers=8"
+				},
+				{
+					"@type": "dcat:Distribution",
+					"title": "KML",
+					"format": "KML",
+					"mediaType": "application/vnd.google-earth.kml+xml",
+					"accessURL": "https://ville-d-evian-opendata-ville-evian.hub.arcgis.com/api/download/v1/items/cb6aa94ccb21492396e799d1ad282c02/kml?layers=8"
+				}
+			],
+			"theme": [
+				"geospatial"
+			]
+		},
+		{
+			"@type": "dcat:Dataset",
+			"identifier": "https://www.arcgis.com/home/item.html?id=b7c181ff568344daae2a77bafbcc7225&sublayer=0",
+			"landingPage": "https://ville-d-evian-opendata-ville-evian.hub.arcgis.com/maps/b7c181ff568344daae2a77bafbcc7225_0",
+			"title": "defibrillateurs",
+			"description": "<DIV STYLE=\"text-align:Left;\"><DIV><DIV><P><SPAN>défibrillateur</SPAN></P></DIV></DIV></DIV>",
+			"keyword": [
+				"sante"
+			],
+			"issued": "2022-07-29T10:07:44.000Z",
+			"modified": "2023-03-13T14:34:36.923Z",
+			"publisher": {
+				"name": "Mairie Evian-les-Bains"
+			},
+			"contactPoint": {
+				"@type": "vcard:Contact",
+				"fn": "sigevian"
+			},
+			"accessLevel": "public",
+			"spatial": "6.5664,46.3889,6.6088,46.4039",
+			"license": "<DIV STYLE=\"text-align:Left;\"><DIV><DIV><P><SPAN>Soumis à la licence Etalab</SPAN></P></DIV></DIV></DIV>",
+			"distribution": [
+				{
+					"@type": "dcat:Distribution",
+					"title": "ArcGIS Hub Dataset",
+					"format": "Web Page",
+					"mediaType": "text/html",
+					"accessURL": "https://ville-d-evian-opendata-ville-evian.hub.arcgis.com/maps/b7c181ff568344daae2a77bafbcc7225_0"
+				},
+				{
+					"@type": "dcat:Distribution",
+					"title": "ArcGIS GeoService",
+					"format": "ArcGIS GeoServices REST API",
+					"mediaType": "application/json",
+					"accessURL": "https://services.arcgis.com/Hbks8qlTugfLqQka/arcgis/rest/services/defibrillateurs/FeatureServer/0"
+				},
+				{
+					"@type": "dcat:Distribution",
+					"title": "CSV",
+					"format": "CSV",
+					"mediaType": "text/csv",
+					"accessURL": "https://ville-d-evian-opendata-ville-evian.hub.arcgis.com/api/download/v1/items/b7c181ff568344daae2a77bafbcc7225/csv?layers=0"
+				},
+				{
+					"@type": "dcat:Distribution",
+					"title": "GeoJSON",
+					"format": "GeoJSON",
+					"mediaType": "application/vnd.geo+json",
+					"accessURL": "https://ville-d-evian-opendata-ville-evian.hub.arcgis.com/api/download/v1/items/b7c181ff568344daae2a77bafbcc7225/geojson?layers=0"
+				},
+				{
+					"@type": "dcat:Distribution",
+					"title": "Shapefile",
+					"format": "ZIP",
+					"mediaType": "application/zip",
+					"accessURL": "https://ville-d-evian-opendata-ville-evian.hub.arcgis.com/api/download/v1/items/b7c181ff568344daae2a77bafbcc7225/shapefile?layers=0"
+				},
+				{
+					"@type": "dcat:Distribution",
+					"title": "KML",
+					"format": "KML",
+					"mediaType": "application/vnd.google-earth.kml+xml",
+					"accessURL": "https://ville-d-evian-opendata-ville-evian.hub.arcgis.com/api/download/v1/items/b7c181ff568344daae2a77bafbcc7225/kml?layers=0"
+				}
+			],
+			"theme": [
+				"geospatial"
+			]
+		},
+		{
+			"@type": "dcat:Dataset",
+			"identifier": "https://www.arcgis.com/home/item.html?id=dcc09451ea4f4c1398584374120137a0&sublayer=6",
+			"landingPage": "https://ville-d-evian-opendata-ville-evian.hub.arcgis.com/maps/dcc09451ea4f4c1398584374120137a0_6",
+			"title": "places de marche",
+			"description": "<DIV STYLE=\"text-align:Left;\"><DIV><DIV><P><SPAN>Zone et description des emplacements du marché à Evian-les-Bains</SPAN></P></DIV></DIV></DIV>",
+			"keyword": [
+				"environnement"
+			],
+			"issued": "2022-07-29T10:06:57.000Z",
+			"modified": "2024-02-09T14:33:50.766Z",
+			"publisher": {
+				"name": "Ville d'Evian-les-Bains"
+			},
+			"contactPoint": {
+				"@type": "vcard:Contact",
+				"fn": "sigevian"
+			},
+			"accessLevel": "public",
+			"spatial": "6.5873,46.4006,6.5889,46.4020",
+			"license": "",
+			"distribution": [
+				{
+					"@type": "dcat:Distribution",
+					"title": "ArcGIS Hub Dataset",
+					"format": "Web Page",
+					"mediaType": "text/html",
+					"accessURL": "https://ville-d-evian-opendata-ville-evian.hub.arcgis.com/maps/dcc09451ea4f4c1398584374120137a0_6"
+				},
+				{
+					"@type": "dcat:Distribution",
+					"title": "ArcGIS GeoService",
+					"format": "ArcGIS GeoServices REST API",
+					"mediaType": "application/json",
+					"accessURL": "https://services.arcgis.com/Hbks8qlTugfLqQka/arcgis/rest/services/place_de_marche/FeatureServer/6"
+				},
+				{
+					"@type": "dcat:Distribution",
+					"title": "CSV",
+					"format": "CSV",
+					"mediaType": "text/csv",
+					"accessURL": "https://ville-d-evian-opendata-ville-evian.hub.arcgis.com/api/download/v1/items/dcc09451ea4f4c1398584374120137a0/csv?layers=6"
+				},
+				{
+					"@type": "dcat:Distribution",
+					"title": "GeoJSON",
+					"format": "GeoJSON",
+					"mediaType": "application/vnd.geo+json",
+					"accessURL": "https://ville-d-evian-opendata-ville-evian.hub.arcgis.com/api/download/v1/items/dcc09451ea4f4c1398584374120137a0/geojson?layers=6"
+				},
+				{
+					"@type": "dcat:Distribution",
+					"title": "Shapefile",
+					"format": "ZIP",
+					"mediaType": "application/zip",
+					"accessURL": "https://ville-d-evian-opendata-ville-evian.hub.arcgis.com/api/download/v1/items/dcc09451ea4f4c1398584374120137a0/shapefile?layers=6"
+				},
+				{
+					"@type": "dcat:Distribution",
+					"title": "KML",
+					"format": "KML",
+					"mediaType": "application/vnd.google-earth.kml+xml",
+					"accessURL": "https://ville-d-evian-opendata-ville-evian.hub.arcgis.com/api/download/v1/items/dcc09451ea4f4c1398584374120137a0/kml?layers=6"
+				}
+			],
+			"theme": [
+				"geospatial"
+			]
+		},
+		{
+			"@type": "dcat:Dataset",
+			"identifier": "https://www.arcgis.com/home/item.html?id=94a712dd0d1747c1b899a58f927a2566&sublayer=4",
+			"landingPage": "https://ville-d-evian-opendata-ville-evian.hub.arcgis.com/maps/94a712dd0d1747c1b899a58f927a2566_4",
+			"title": "places PMR",
+			"description": "<DIV STYLE=\"text-align:Left;\"><DIV><DIV><P><SPAN>Emplacement et description des places PMR</SPAN></P></DIV></DIV></DIV>",
+			"keyword": [
+				"mobilite"
+			],
+			"issued": "2022-07-29T09:09:31.000Z",
+			"modified": "2022-08-23T11:34:50.466Z",
+			"publisher": {
+				"name": "Ville Evian-les-Bains"
+			},
+			"contactPoint": {
+				"@type": "vcard:Contact",
+				"fn": "sigevian"
+			},
+			"accessLevel": "public",
+			"spatial": "6.5662,46.3885,6.6130,46.4045",
+			"license": "",
+			"distribution": [
+				{
+					"@type": "dcat:Distribution",
+					"title": "ArcGIS Hub Dataset",
+					"format": "Web Page",
+					"mediaType": "text/html",
+					"accessURL": "https://ville-d-evian-opendata-ville-evian.hub.arcgis.com/maps/94a712dd0d1747c1b899a58f927a2566_4"
+				},
+				{
+					"@type": "dcat:Distribution",
+					"title": "ArcGIS GeoService",
+					"format": "ArcGIS GeoServices REST API",
+					"mediaType": "application/json",
+					"accessURL": "https://services.arcgis.com/Hbks8qlTugfLqQka/arcgis/rest/services/places_PMR/FeatureServer/4"
+				},
+				{
+					"@type": "dcat:Distribution",
+					"title": "CSV",
+					"format": "CSV",
+					"mediaType": "text/csv",
+					"accessURL": "https://ville-d-evian-opendata-ville-evian.hub.arcgis.com/api/download/v1/items/94a712dd0d1747c1b899a58f927a2566/csv?layers=4"
+				},
+				{
+					"@type": "dcat:Distribution",
+					"title": "GeoJSON",
+					"format": "GeoJSON",
+					"mediaType": "application/vnd.geo+json",
+					"accessURL": "https://ville-d-evian-opendata-ville-evian.hub.arcgis.com/api/download/v1/items/94a712dd0d1747c1b899a58f927a2566/geojson?layers=4"
+				},
+				{
+					"@type": "dcat:Distribution",
+					"title": "Shapefile",
+					"format": "ZIP",
+					"mediaType": "application/zip",
+					"accessURL": "https://ville-d-evian-opendata-ville-evian.hub.arcgis.com/api/download/v1/items/94a712dd0d1747c1b899a58f927a2566/shapefile?layers=4"
+				},
+				{
+					"@type": "dcat:Distribution",
+					"title": "KML",
+					"format": "KML",
+					"mediaType": "application/vnd.google-earth.kml+xml",
+					"accessURL": "https://ville-d-evian-opendata-ville-evian.hub.arcgis.com/api/download/v1/items/94a712dd0d1747c1b899a58f927a2566/kml?layers=4"
+				}
+			],
+			"theme": [
+				"geospatial"
+			]
+		},
+		{
+			"@type": "dcat:Dataset",
+			"identifier": "https://www.arcgis.com/home/item.html?id=92a676a42a864c4baf58856e445b6d41",
+			"landingPage": "https://ville-d-evian-opendata-ville-evian.hub.arcgis.com/documents/92a676a42a864c4baf58856e445b6d41_1231",
+			"title": "Publication actes dématérialisés",
+			"description": "",
+			"keyword": [
+				"OpenData"
+			],
+			"issued": "2023-10-30T08:48:06.000Z",
+			"modified": "2023-10-30T08:56:01.000Z",
+			"publisher": {
+				"name": "sigevian"
+			},
+			"contactPoint": {
+				"@type": "vcard:Contact",
+				"fn": "sigevian"
+			},
+			"accessLevel": "public",
+			"spatial": "{{extent:computeSpatialProperty}}",
+			"license": "",
+			"distribution": [
+				{
+					"@type": "dcat:Distribution",
+					"title": "ArcGIS Hub Dataset",
+					"format": "Web Page",
+					"mediaType": "text/html",
+					"accessURL": "https://ville-d-evian-opendata-ville-evian.hub.arcgis.com/documents/92a676a42a864c4baf58856e445b6d41_1231"
+				},
+				{
+					"@type": "dcat:Distribution",
+					"title": "ArcGIS GeoService",
+					"format": "ArcGIS GeoServices REST API",
+					"mediaType": "application/json",
+					"accessURL": "https://publiact.fr/company/1231"
+				}
+			],
+			"theme": [
+				"geospatial"
+			]
+		},
+		{
+			"@type": "dcat:Dataset",
+			"identifier": "https://www.arcgis.com/home/item.html?id=856b7eb926a14f4cb34d95597bafb610",
+			"landingPage": "https://ville-d-evian-opendata-ville-evian.hub.arcgis.com/documents/856b7eb926a14f4cb34d95597bafb610",
+			"title": "Carte piéton concertation",
+			"description": "",
+			"keyword": [
+				"evian",
+				"mobilite"
+			],
+			"issued": "2022-07-25T14:30:43.000Z",
+			"modified": "2022-08-01T09:26:34.000Z",
+			"publisher": {
+				"name": "sigevian"
+			},
+			"contactPoint": {
+				"@type": "vcard:Contact",
+				"fn": "sigevian"
+			},
+			"accessLevel": "public",
+			"spatial": "{{extent:computeSpatialProperty}}",
+			"license": "",
+			"distribution": [
+				{
+					"@type": "dcat:Distribution",
+					"title": "ArcGIS Hub Dataset",
+					"format": "Web Page",
+					"mediaType": "text/html",
+					"accessURL": "https://ville-d-evian-opendata-ville-evian.hub.arcgis.com/documents/856b7eb926a14f4cb34d95597bafb610"
+				}
+			],
+			"theme": [
+				"geospatial"
+			]
+		},
+		{
+			"@type": "dcat:Dataset",
+			"identifier": "https://www.arcgis.com/home/item.html?id=8acd3deee8d042538e21925d0640f199",
+			"landingPage": "https://ville-d-evian-opendata-ville-evian.hub.arcgis.com/documents/8acd3deee8d042538e21925d0640f199",
+			"title": "Carte vélo finale",
+			"description": "",
+			"keyword": [
+				"evian",
+				"mobilite"
+			],
+			"issued": "2022-07-25T14:30:41.000Z",
+			"modified": "2022-07-28T13:29:26.000Z",
+			"publisher": {
+				"name": "sigevian"
+			},
+			"contactPoint": {
+				"@type": "vcard:Contact",
+				"fn": "sigevian"
+			},
+			"accessLevel": "public",
+			"spatial": "{{extent:computeSpatialProperty}}",
+			"license": "",
+			"distribution": [
+				{
+					"@type": "dcat:Distribution",
+					"title": "ArcGIS Hub Dataset",
+					"format": "Web Page",
+					"mediaType": "text/html",
+					"accessURL": "https://ville-d-evian-opendata-ville-evian.hub.arcgis.com/documents/8acd3deee8d042538e21925d0640f199"
+				}
+			],
+			"theme": [
+				"geospatial"
+			]
+		}
+	]
+}

udata/harvest/tests/test_dcat_backend.py

@@ -1,21 +1,22 @@
+from datetime import date
 import logging
 import os
-from flask import current_app
+import re
 
 import pytest
 
-from datetime import date
 import boto3
+from flask import current_app
 import xml.etree.ElementTree as ET
-from udata.harvest.models import HarvestJob
 
+from udata.harvest.models import HarvestJob
 from udata.models import Dataset
 from udata.core.organization.factories import OrganizationFactory
 from udata.core.dataset.factories import LicenseFactory, ResourceSchemaMockData
 from udata.storage.s3 import get_from_json
 
 from .factories import HarvestSourceFactory
-from ..backends.dcat import URIS_TO_REPLACE
+from ..backends.dcat import URIS_TO_REPLACE, CswIso19139DcatBackend
 from .. import actions
 
 log = logging.getLogger(__name__)
@@ -160,6 +161,21 @@ class DcatBackendTest:
         assert len(datasets['1'].resources) == 2
         assert len(datasets['2'].resources) == 2
 
+    def test_harvest_literal_spatial(self, rmock):
+        url = mock_dcat(rmock, 'evian.json')
+        org = OrganizationFactory()
+        source = HarvestSourceFactory(backend='dcat',
+                                      url=url,
+                                      organization=org)
+        
+        actions.run(source.slug)
+
+        datasets = {d.harvest.dct_identifier: d for d in Dataset.objects}
+        assert len(datasets) == 8
+        assert datasets['https://www.arcgis.com/home/item.html?id=f6565516d1354383b25793e630cf3f2b&sublayer=5'].spatial is not None
+        assert datasets['https://www.arcgis.com/home/item.html?id=f6565516d1354383b25793e630cf3f2b&sublayer=5'].spatial.geom == {'type': 'MultiPolygon', 'coordinates': [[[[6.5735, 46.3912], [6.6069, 46.3912], [6.6069, 46.4028], [6.5735, 46.4028], [6.5735, 46.3912]]]]}
+
+
     @pytest.mark.skip(reason="Mocking S3 requires `moto` which is not available for our current Python 3.7. We can manually test it.")
     @pytest.mark.options(SCHEMA_CATALOG_URL='https://example.com/schemas', HARVEST_JOBS_RETENTION_DAYS=0)
     # @mock_s3
@@ -195,8 +211,8 @@ class DcatBackendTest:
         assert resources_by_title['Resource 1-2'].schema.name == None
         assert resources_by_title['Resource 1-2'].schema.version == None
 
-        assert datasets['2'].schema.name == None
-        assert datasets['2'].schema.url == 'https://www.ecologie.gouv.fr/sites/default/files/R%C3%A9glementation%20IRVE.pdf'
+        assert datasets['2'].schema.name == 'RGF93 / Lambert-93 (EPSG:2154)'
+        assert datasets['2'].schema.url == 'http://inspire.ec.europa.eu/glossary/SpatialReferenceSystem'
         resources_by_title = { resource['title']: resource for resource in datasets['2'].resources }
 
         # Unknown schema are kept as they were provided
@@ -265,8 +281,8 @@ class DcatBackendTest:
         assert resources_by_title['Resource 1-2'].schema.name == None
         assert resources_by_title['Resource 1-2'].schema.version == None
 
-        assert datasets['2'].schema.name == None
-        assert datasets['2'].schema.url == 'https://www.ecologie.gouv.fr/sites/default/files/R%C3%A9glementation%20IRVE.pdf'
+        assert datasets['2'].schema.name == 'RGF93 / Lambert-93 (EPSG:2154)'
+        assert datasets['2'].schema.url == 'http://inspire.ec.europa.eu/glossary/SpatialReferenceSystem'
         resources_by_title = { resource['title']: resource for resource in datasets['2'].resources }
 
         # Unknown schema are kept as they were provided
@@ -614,7 +630,7 @@ class DcatBackendTest:
 
 
 @pytest.mark.usefixtures('clean_db')
-@pytest.mark.options(PLUGINS=['csw-dcat'])
+@pytest.mark.options(PLUGINS=['csw'])
 class CswDcatBackendTest:
 
     def test_geonetworkv4(self, rmock):
@@ -662,3 +678,60 @@ class CswDcatBackendTest:
 
         assert 'User-Agent' in get_mock.last_request.headers
         assert get_mock.last_request.headers['User-Agent'] == 'uData/0.1 csw-dcat'
+
+
+@pytest.mark.usefixtures('clean_db')
+@pytest.mark.options(PLUGINS=['csw'])
+class CswIso19139DcatBackendTest:
+
+    def test_geo2france(self, rmock):
+
+        with open(os.path.join(CSW_DCAT_FILES_DIR, "XSLT.xml"), "r") as f:
+            xslt = f.read()
+        url = mock_csw_pagination(rmock, 'geonetwork/srv/eng/csw.rdf', 'geonetwork-iso-page-{}.xml')
+        rmock.get(CswIso19139DcatBackend.XSL_URL, text=xslt)
+        org = OrganizationFactory()
+        source = HarvestSourceFactory(backend='csw-iso-19139',
+                                      url=url,
+                                      organization=org)
+
+        actions.run(source.slug)
+
+        source.reload()
+
+        job = source.get_last_job()
+        assert len(job.items) == 6
+
+        datasets = {d.harvest.dct_identifier: d for d in Dataset.objects}
+
+        assert len(datasets) == 6
+
+        # First dataset
+        # dataset identifier is gmd:RS_Identifier > gmd:codeSpace + gmd:code
+        dataset = datasets['http://catalogue.geo-ide.developpement-durable.gouv.fr/fr-120066022-orphan-residentifier-140d31c6-643d-42a9-85df-2737a118e144']
+        assert dataset.title == "Plan local d'urbanisme de la commune de Cartigny"
+        assert dataset.description == "Le présent standard de données COVADIS concerne les documents de plans locaux d'urbanisme (PLU) et les plans d'occupation des sols (POS qui valent PLU)."
+        assert set(dataset.tags) == set([
+            'amenagement-urbanisme-zonages-planification', 'cartigny',
+            'document-durbanisme', 'donnees-ouvertes', 'plu', 'usage-des-sols'
+        ])
+        assert dataset.harvest.created_at.date() == date(2017, 10, 7)
+        assert dataset.spatial.geom == {'type': 'MultiPolygon', 'coordinates':
+            [[[[3.28133559, 50.48188019], [1.31279111, 50.48188019], [1.31279111, 49.38547516], [3.28133559, 49.38547516], [3.28133559, 50.48188019]]]]
+        }
+        assert dataset.contact_point.name == 'DDTM 80 (Direction Départementale des Territoires et de la Mer de la Somme)'
+        assert dataset.contact_point.email == 'ddtm-sap-bsig@somme.gouv.fr'
+
+        # License is not properly mapped in XSLT conversion
+        assert dataset.license is None
+
+        # Distributions don't get properly mapped to distribution with this XSLT if missing CI_OnLineFunctionCode.
+        # A CI_OnLineFunctionCode was added explicitely on one of the Online Resources.
+        # (See mapping at: https://semiceu.github.io/GeoDCAT-AP/releases/2.0.0/#resource-locator---on-line-resource)
+        assert len(dataset.resources) == 1
+        resource = dataset.resources[0]
+        assert resource.title == 'Téléchargement direct du lot et des documents associés'
+        assert resource.url == 'http://atom.geo-ide.developpement-durable.gouv.fr/atomArchive/GetResource?id=fr-120066022-ldd-cab63273-b3ae-4e8a-ae1c-6192e45faa94&datasetAggregate=true'
+        
+        # Sadly resource format is parsed as a blank node. Format parsing should be improved.
+        assert re.match(r'n[0-9a-f]{32}', resource.format)

udata/migrations/2024-03-22-migrate-activity-kwargs-to-extras.py

@@ -0,0 +1,16 @@
+"""
+Migrates `Activity.kwargs` to `Activity.extras`.
+"""
+import logging
+
+from mongoengine.connection import get_db
+
+log = logging.getLogger(__name__)
+
+def migrate(db):
+    log.info('Processing activity objects')
+
+    result = get_db().activity.update_many(filter={}, update={'$rename': {'kwargs': 'extras'}})
+
+    log.info(f'{result.modified_count} activity object(s) migrated')
+    log.info('Done')