udata 13.0.0__py3-none-any.whl → 13.0.1.dev2__py3-none-any.whl

udata/harvest/tests/test_dcat_backend.py

@@ -714,6 +714,48 @@ class DcatBackendTest(PytestOnlyDBTestCase):
         )  # noqa
         assert dataset.harvest.last_update.date() == date.today()
 
+    def test_datara_extended_roles_foaf(self, rmock):
+        # Converted manually from ISO-19139 using SEMICeu XSLT (tag geodcat-ap-2.0.0)
+        url = mock_dcat(rmock, "datara--5a26b0f6-0ccf-46ad-ac58-734054b91977.rdf.xml")
+        org = OrganizationFactory()
+        source = HarvestSourceFactory(backend="dcat", url=url, organization=org)
+        actions.run(source)
+        dataset = Dataset.objects.filter(organization=org).first()
+
+        assert dataset is not None
+        assert len(dataset.contact_points) == 2
+
+        assert dataset.contact_points[0].name == "IGN"
+        assert dataset.contact_points[0].email == "sav.bd@ign.fr"
+        assert dataset.contact_points[0].role == "rightsHolder"
+
+        assert dataset.contact_points[1].name == "Administrateur de Données"
+        assert dataset.contact_points[1].email == "sig.dreal-ara@developpement-durable.gouv.fr"
+        assert dataset.contact_points[1].role == "user"
+
+    def test_datara_extended_roles_vcard(self, rmock):
+        # Converted manually from ISO-19139 using SEMICeu XSLT (tag geodcat-ap-2.0.0)
+        url = mock_dcat(rmock, "datara--f40c3860-7236-4b30-a141-23b8ae33f7b2.rdf.xml")
+        org = OrganizationFactory()
+        source = HarvestSourceFactory(backend="dcat", url=url, organization=org)
+        actions.run(source)
+        dataset = Dataset.objects.filter(organization=org).first()
+
+        assert dataset is not None
+        assert len(dataset.contact_points) == 3
+
+        assert dataset.contact_points[0].name == "Administrateur de Données"
+        assert dataset.contact_points[0].email == "sig.dreal-ara@developpement-durable.gouv.fr"
+        assert dataset.contact_points[0].role == "contact"
+
+        assert dataset.contact_points[1].name == "Jean-Michel GENIS"
+        assert dataset.contact_points[1].email == "jm.genis@cbn-alpin.fr"
+        assert dataset.contact_points[1].role == "rightsHolder"
+
+        assert dataset.contact_points[2].name == "Conservatoire Botanique National Massif Central"
+        assert dataset.contact_points[2].email == "Benoit.Renaux@cbnmc.fr"
+        assert dataset.contact_points[2].role == "rightsHolder"
+
     def test_udata_xml_catalog(self, rmock):
         LicenseFactory(id="fr-lo", title="Licence ouverte / Open Licence")
         url = mock_dcat(rmock, "udata.xml")

udata/migrations/2025-10-21-remove-ckan-harvest-modified-at.py

@@ -0,0 +1,28 @@
+"""
+This migration empties harvest.modified_at field in the case of CKAN datasets.
+Indeed, the value that was stored in this field was the *metadata* modification data
+and not the *data* one, contrary to other backends.
+"""
+
+import logging
+
+import click
+
+from udata.core.dataset.models import Dataset
+
+log = logging.getLogger(__name__)
+
+
+def migrate(db):
+    datasets = Dataset.objects(harvest__backend="CKAN", harvest__modified_at__exists=True)
+    count = datasets.count()
+
+    with click.progressbar(datasets, length=count) as datasets:
+        for dataset in datasets:
+            dataset.harvest.modified_at = None
+            try:
+                dataset.save()
+            except Exception as err:
+                log.error(f"Cannot save dataset {dataset.id} {err}")
+    log.info(f"Updated {count} datasets")
+    log.info("Done")

udata/rdf.py

@@ -136,15 +136,21 @@ INSPIRE_GEMET_SCHEME_URIS = [
 
 AGENT_ROLE_TO_RDF_PREDICATE = {
     "contact": DCAT.contactPoint,
-    "publisher": DCT.publisher,
     "creator": DCT.creator,
+    "publisher": DCT.publisher,
+    "rightsHolder": DCT.rightsHolder,
+    "custodian": GEODCAT.custodian,
+    "distributor": GEODCAT.distributor,
+    "originator": GEODCAT.originator,
+    "principalInvestigator": GEODCAT.principalInvestigator,
+    "processor": GEODCAT.processor,
+    "resourceProvider": GEODCAT.resourceProvider,
+    "user": GEODCAT.user,
 }
 
 # Map rdf contact point entity to role
 CONTACT_POINT_ENTITY_TO_ROLE = {
-    DCAT.contactPoint: "contact",
-    DCT.publisher: "publisher",
-    DCT.creator: "creator",
+    predicate: role for role, predicate in AGENT_ROLE_TO_RDF_PREDICATE.items()
 }
 
 
@@ -353,6 +359,8 @@ def themes_from_rdf(rdf):
 
 
 def contact_points_from_rdf(rdf, prop, role, dataset):
+    if not dataset.organization and not dataset.owner:
+        return
     for contact_point in rdf.objects(prop):
         # Read contact point information
         if isinstance(contact_point, Literal):
@@ -365,7 +373,7 @@ def contact_points_from_rdf(rdf, prop, role, dataset):
             email = (
                 rdf_value(contact_point, VCARD.hasEmail)
                 or rdf_value(contact_point, VCARD.email)
-                or rdf_value(contact_point, DCAT.email)
+                or None
             )
             email = email.replace("mailto:", "").strip() if email else None
             contact_form = rdf_value(contact_point, VCARD.hasUrl)
@@ -384,8 +392,6 @@ def contact_points_from_rdf(rdf, prop, role, dataset):
         #         continue
 
         # Create of get contact point object
-        if not dataset.organization and not dataset.owner:
-            continue
         org_or_owner = {}
         if dataset.organization:
             org_or_owner = {"organization": dataset.organization}
@@ -420,14 +426,25 @@ def contact_points_to_rdf(contacts, graph=None):
             id = BNode()
 
         node = graph.resource(id)
-        node.set(RDF.type, VCARD.Kind)
-        if contact.name:
-            node.set(VCARD.fn, Literal(contact.name))
-        if contact.email:
-            node.set(VCARD.hasEmail, URIRef(f"mailto:{contact.email}"))
-        if contact.contact_form:
-            node.set(VCARD.hasUrl, URIRef(contact.contact_form))
-        yield node, AGENT_ROLE_TO_RDF_PREDICATE.get(contact.role, DCAT.contactPoint)
+        role = AGENT_ROLE_TO_RDF_PREDICATE.get(contact.role, DCAT.contactPoint)
+        # GeoDCAT-AP spec: Only contactPoint is a VCARD.Kind (like in DCAT). Other roles are FOAF.Agent.
+        if role == DCAT.contactPoint:
+            node.set(RDF.type, VCARD.Kind)
+            if contact.name:
+                node.set(VCARD.fn, Literal(contact.name))
+            if contact.email:
+                node.set(VCARD.hasEmail, URIRef(f"mailto:{contact.email}"))
+            if contact.contact_form:
+                node.set(VCARD.hasUrl, URIRef(contact.contact_form))
+        else:
+            node.set(RDF.type, FOAF.Agent)
+            node.set(FOAF.name, Literal(contact.name))
+            if contact.email:
+                node.set(FOAF.mbox, URIRef(f"mailto:{contact.email}"))
+            if contact.contact_form:
+                node.set(FOAF.page, URIRef(contact.contact_form))
+
+        yield node, role
 
 
 def primary_topic_identifier_from_rdf(graph: Graph, resource: RdfResource):

udata/tests/dataset/test_dataset_rdf.py

@@ -5,7 +5,7 @@ import pytest
 import requests
 from flask import url_for
 from rdflib import BNode, Graph, Literal, Namespace, URIRef
-from rdflib.namespace import FOAF, RDF, RDFS
+from rdflib.namespace import FOAF, ORG, RDF, RDFS
 from rdflib.resource import Resource as RdfResource
 
 from udata.core.contact_point.factories import ContactPointFactory
@@ -166,8 +166,8 @@ class DatasetToRdfTest(PytestOnlyAPITestCase):
         d = dataset_to_rdf(dataset)
 
         contact_rdf = d.value(DCT.publisher)
-        assert contact_rdf.value(RDF.type).identifier == VCARD.Kind
-        assert contact_rdf.value(VCARD.fn) == Literal("Publisher Contact")
+        assert contact_rdf.value(RDF.type).identifier == FOAF.Agent
+        assert contact_rdf.value(FOAF.name) == Literal("Publisher Contact")
 
         org_rdf = d.value(GEODCAT.distributor)
         assert org_rdf.value(RDF.type).identifier == FOAF.Organization
@@ -514,6 +514,194 @@ class RdfToDatasetTest(PytestOnlyDBTestCase):
         assert isinstance(dataset, Dataset)
         assert dataset.harvest.modified_at is None
 
+    def test_contact_point_individual_vcard(self):
+        g = Graph()
+        node = URIRef("https://test.org/dataset")
+        g.set((node, RDF.type, DCAT.Dataset))
+        g.set((node, DCT.identifier, Literal(faker.uuid4())))
+        g.set((node, DCT.title, Literal(faker.sentence())))
+
+        contact = BNode()
+        g.add((contact, RDF.type, VCARD.Individual))
+        g.add((contact, VCARD.fn, Literal("foo")))
+        g.add((contact, VCARD.email, Literal("foo@example.com")))
+        g.add((node, DCAT.contactPoint, contact))
+
+        # Dataset needs an owner/organization for contact_points_from_rdf() to work
+        d = DatasetFactory.build()
+        d.organization = OrganizationFactory(name="organization")
+
+        dataset = dataset_from_rdf(g, d)
+        dataset.validate()
+
+        assert len(dataset.contact_points) == 1
+        assert dataset.contact_points[0].role == "contact"
+        assert dataset.contact_points[0].name == "foo"
+        assert dataset.contact_points[0].email == "foo@example.com"
+
+    def test_contact_point_individual_foaf(self):
+        g = Graph()
+        node = URIRef("https://test.org/dataset")
+        g.set((node, RDF.type, DCAT.Dataset))
+        g.set((node, DCT.identifier, Literal(faker.uuid4())))
+        g.set((node, DCT.title, Literal(faker.sentence())))
+
+        contact = BNode()
+        contact_name = Literal("foo")
+        contact_email = Literal("foo@example.com")
+        g.add((contact, RDF.type, FOAF.Person))
+        g.add((contact, FOAF.name, contact_name))
+        g.add((contact, FOAF.mbox, contact_email))
+        g.add((node, DCT.creator, contact))
+
+        # Dataset needs an owner/organization for contact_points_from_rdf() to work
+        d = DatasetFactory.build()
+        d.organization = OrganizationFactory(name="organization")
+
+        dataset = dataset_from_rdf(g, d)
+        dataset.validate()
+
+        assert len(dataset.contact_points) == 1
+        assert dataset.contact_points[0].role == "creator"
+        assert dataset.contact_points[0].name == "foo"
+        assert dataset.contact_points[0].email == "foo@example.com"
+
+    def test_contact_point_organization_vcard(self):
+        g = Graph()
+        node = URIRef("https://test.org/dataset")
+        g.set((node, RDF.type, DCAT.Dataset))
+        g.set((node, DCT.identifier, Literal(faker.uuid4())))
+        g.set((node, DCT.title, Literal(faker.sentence())))
+
+        contact = BNode()
+        g.add((contact, RDF.type, VCARD.Organization))
+        g.add((contact, VCARD.fn, Literal("foo")))
+        g.add((contact, VCARD.email, Literal("foo@example.com")))
+        g.add((node, DCAT.contactPoint, contact))
+
+        # Dataset needs an owner/organization for contact_points_from_rdf() to work
+        d = DatasetFactory.build()
+        d.organization = OrganizationFactory(name="organization")
+
+        dataset = dataset_from_rdf(g, d)
+        dataset.validate()
+
+        assert len(dataset.contact_points) == 1
+        assert dataset.contact_points[0].role == "contact"
+        assert dataset.contact_points[0].name == "foo"
+        assert dataset.contact_points[0].email == "foo@example.com"
+
+    def test_contact_point_organization_foaf(self):
+        g = Graph()
+        node = URIRef("https://test.org/dataset")
+        g.set((node, RDF.type, DCAT.Dataset))
+        g.set((node, DCT.identifier, Literal(faker.uuid4())))
+        g.set((node, DCT.title, Literal(faker.sentence())))
+
+        contact = BNode()
+        g.add((contact, RDF.type, FOAF.Organization))
+        g.add((contact, FOAF.name, Literal("foo")))
+        g.add((contact, FOAF.mbox, Literal("foo@example.com")))
+        g.add((node, DCT.creator, contact))
+
+        # Dataset needs an owner/organization for contact_points_from_rdf() to work
+        d = DatasetFactory.build()
+        d.organization = OrganizationFactory(name="organization")
+
+        dataset = dataset_from_rdf(g, d)
+        dataset.validate()
+
+        assert len(dataset.contact_points) == 1
+        assert dataset.contact_points[0].role == "creator"
+        assert dataset.contact_points[0].name == "foo"
+        assert dataset.contact_points[0].email == "foo@example.com"
+
+    def test_contact_point_organization_member_vcard(self):
+        g = Graph()
+        node = URIRef("https://test.org/dataset")
+        g.set((node, RDF.type, DCAT.Dataset))
+        g.set((node, DCT.identifier, Literal(faker.uuid4())))
+        g.set((node, DCT.title, Literal(faker.sentence())))
+
+        contact = BNode()
+        g.add((contact, RDF.type, VCARD.Organization))
+        g.add((contact, VCARD.fn, Literal("foo")))
+        g.add((contact, VCARD["organization-name"], Literal("bar")))
+        g.add((contact, VCARD.email, Literal("foo@example.com")))
+        g.add((node, DCAT.contactPoint, contact))
+
+        # Dataset needs an owner/organization for contact_points_from_rdf() to work
+        d = DatasetFactory.build()
+        d.organization = OrganizationFactory(name="organization")
+
+        dataset = dataset_from_rdf(g, d)
+        dataset.validate()
+
+        assert len(dataset.contact_points) == 1
+        assert dataset.contact_points[0].role == "contact"
+        assert dataset.contact_points[0].name == "foo"
+        assert dataset.contact_points[0].email == "foo@example.com"
+
+    def test_contact_point_organization_member_foaf(self):
+        g = Graph()
+        node = URIRef("https://test.org/dataset")
+        g.set((node, RDF.type, DCAT.Dataset))
+        g.set((node, DCT.identifier, Literal(faker.uuid4())))
+        g.set((node, DCT.title, Literal(faker.sentence())))
+
+        org = BNode()
+        g.add((org, RDF.type, FOAF.Organization))
+        g.add((org, FOAF.name, Literal("bar")))
+        g.add((org, FOAF.mbox, Literal("bar@example.com")))
+        contact = BNode()
+        g.add((contact, RDF.type, FOAF.Person))
+        g.add((contact, FOAF.name, Literal("foo")))
+        g.add((contact, FOAF.mbox, Literal("foo@example.com")))
+        g.add((contact, ORG.memberOf, org))
+        g.add((node, DCT.creator, contact))
+
+        # Dataset needs an owner/organization for contact_points_from_rdf() to work
+        d = DatasetFactory.build()
+        d.organization = OrganizationFactory(name="organization")
+
+        dataset = dataset_from_rdf(g, d)
+        dataset.validate()
+
+        assert len(dataset.contact_points) == 1
+        assert dataset.contact_points[0].role == "creator"
+        assert dataset.contact_points[0].name == "foo"
+        assert dataset.contact_points[0].email == "foo@example.com"
+
+    def test_contact_point_organization_member_foaf_no_mail(self):
+        g = Graph()
+        node = URIRef("https://test.org/dataset")
+        g.set((node, RDF.type, DCAT.Dataset))
+        g.set((node, DCT.identifier, Literal(faker.uuid4())))
+        g.set((node, DCT.title, Literal(faker.sentence())))
+
+        org = BNode()
+        g.add((org, RDF.type, FOAF.Organization))
+        g.add((org, FOAF.name, Literal("bar")))
+        # no organization email
+        contact = BNode()
+        g.add((contact, RDF.type, FOAF.Person))
+        g.add((contact, FOAF.name, Literal("foo")))
+        g.add((contact, FOAF.mbox, Literal("foo@example.com")))
+        g.add((contact, ORG.memberOf, org))
+        g.add((node, DCT.creator, contact))
+
+        # Dataset needs an owner/organization for contact_points_from_rdf() to work
+        d = DatasetFactory.build()
+        d.organization = OrganizationFactory(name="organization")
+
+        dataset = dataset_from_rdf(g, d)
+        dataset.validate()
+
+        assert len(dataset.contact_points) == 1
+        assert dataset.contact_points[0].role == "creator"
+        assert dataset.contact_points[0].name == "foo"
+        assert dataset.contact_points[0].email == "foo@example.com"
+
     def test_theme_and_tags(self):
         node = BNode()
         g = Graph()

udata/tests/test_rdf.py

@@ -7,8 +7,9 @@ from rdflib import (
 from udata.models import ContactPoint
 from udata.rdf import (
     ACCEPTED_MIME_TYPES,
+    AGENT_ROLE_TO_RDF_PREDICATE,
     DCAT,
-    DCT,
+    FOAF,
     FORMAT_MAP,
     RDF,
     VCARD,
@@ -103,14 +104,7 @@ class ContactToRdfTest:
             # Default predicate is "contact"
             assert predicate == DCAT.contactPoint
 
-    @pytest.mark.parametrize(
-        "role,predicate",
-        [
-            ("contact", DCAT.contactPoint),
-            ("publisher", DCT.publisher),
-            ("creator", DCT.creator),
-        ],
-    )
+    @pytest.mark.parametrize("role,predicate", AGENT_ROLE_TO_RDF_PREDICATE.items())
     def test_contact_points_to_rdf_roles(self, role, predicate):
         contact = ContactPoint(
             name="Organization contact",
@@ -122,10 +116,20 @@ class ContactToRdfTest:
         contact_rdfs = contact_points_to_rdf([contact], None)
 
         for contact_point, contact_point_predicate in contact_rdfs:
-            assert contact_point.value(RDF.type).identifier == VCARD.Kind
-            assert contact_point.value(VCARD.fn) == Literal("Organization contact")
-            assert contact_point.value(VCARD.hasEmail).identifier == URIRef("mailto:hello@its.me")
-            assert contact_point.value(VCARD.hasUrl).identifier == URIRef(
-                "https://data.support.com"
-            )
             assert contact_point_predicate == predicate
+            if predicate == DCAT.contactPoint:
+                assert contact_point.value(RDF.type).identifier == VCARD.Kind
+                assert contact_point.value(VCARD.fn) == Literal("Organization contact")
+                assert contact_point.value(VCARD.hasEmail).identifier == URIRef(
+                    "mailto:hello@its.me"
+                )
+                assert contact_point.value(VCARD.hasUrl).identifier == URIRef(
+                    "https://data.support.com"
+                )
+            else:
+                assert contact_point.value(RDF.type).identifier == FOAF.Agent
+                assert contact_point.value(FOAF.name) == Literal("Organization contact")
+                assert contact_point.value(FOAF.mbox).identifier == URIRef("mailto:hello@its.me")
+                assert contact_point.value(FOAF.page).identifier == URIRef(
+                    "https://data.support.com"
+                )