udata 10.5.1.dev36034__py2.py3-none-any.whl → 10.5.1.dev36076__py2.py3-none-any.whl

udata/core/dataset/transport.py

@@ -0,0 +1,46 @@
+import requests
+from flask import current_app
+
+from udata.commands import error, success
+from udata.models import Dataset
+from udata.tasks import job
+
+
+def process_dataset(dataset):
+    target_dataset = Dataset.objects(id=dataset["datagouv_id"]).first()
+    if not target_dataset:
+        error(f"Dataset {dataset['id']} not found")
+        return
+    target_dataset.extras["transport:url"] = dataset["page_url"]
+    target_dataset.save()
+
+
+def clear_datasets():
+    nb_datasets = Dataset.objects.filter(
+        **{
+            "extras__transport:url__exists": True,
+        }
+    ).update(
+        **{
+            "unset__extras__transport:url": True,
+        }
+    )
+    success(f"Removed transport:url from {nb_datasets} dataset(s)")
+
+
+@job("map-transport-datasets")
+def map_transport_datasets(self):
+    source = current_app.config.get("TRANSPORT_DATASETS_URL", None)
+    if not source:
+        error("TRANSPORT_DATASETS_URL variable must be set.")
+        return
+
+    response = requests.get(source)
+    if response.status_code != 200:
+        error("Remote platform unreachable.")
+        return
+    results_list = response.json()
+    clear_datasets()
+    for dataset in results_list:
+        process_dataset(dataset)
+    success(f"Done. {len(results_list)} datasets mapped to transport")

udata/harvest/backends/dcat.py

@@ -344,8 +344,6 @@ class CswIso19139DcatBackend(DcatBackend):
 
     ISO_SCHEMA = "http://www.isotc211.org/2005/gmd"
 
-    XSL_URL = "https://raw.githubusercontent.com/datagouv/iso-19139-to-dcat-ap/patch-datagouv/iso-19139-to-dcat-ap.xsl"
-
     def walk_graph(self, url: str, fmt: str) -> Generator[tuple[int, Graph], None, None]:
         """
         Yield all RDF pages as `Graph` from the source
@@ -355,7 +353,8 @@ class CswIso19139DcatBackend(DcatBackend):
         See https://github.com/SEMICeu/iso-19139-to-dcat-ap for more information on the XSLT.
         """
         # Load XSLT
-        xsl = ET.fromstring(self.get(self.XSL_URL).content, parser=SAFE_PARSER)
+        xsl_url = current_app.config["HARVEST_ISO19139_XSL_URL"]
+        xsl = ET.fromstring(self.get(xsl_url).content, parser=SAFE_PARSER)
         transform = ET.XSLT(xsl)
 
         # Start querying and parsing graph

udata/harvest/tests/test_dcat_backend.py

@@ -18,7 +18,7 @@ from udata.rdf import DCAT, RDF, namespace_manager
 from udata.storage.s3 import get_from_json
 
 from .. import actions
-from ..backends.dcat import URIS_TO_REPLACE, CswIso19139DcatBackend
+from ..backends.dcat import URIS_TO_REPLACE
 from .factories import HarvestSourceFactory
 
 log = logging.getLogger(__name__)
@@ -886,7 +886,7 @@ class CswIso19139DcatBackendTest:
         with open(os.path.join(CSW_DCAT_FILES_DIR, "XSLT.xml"), "r") as f:
             xslt = f.read()
         url = mock_csw_pagination(rmock, "geonetwork/srv/eng/csw.rdf", "geonetwork-iso-page-{}.xml")
-        rmock.get(CswIso19139DcatBackend.XSL_URL, text=xslt)
+        rmock.get(current_app.config.get("HARVEST_ISO19139_XSL_URL"), text=xslt)
         org = OrganizationFactory()
         source = HarvestSourceFactory(
             backend="csw-iso-19139",

udata/settings.py

@@ -281,6 +281,8 @@ class Defaults(object):
     HARVEST_GRAPHS_S3_BUCKET = None  # If the catalog is bigger than `HARVEST_MAX_CATALOG_SIZE_IN_MONGO` store the graph inside S3 instead of MongoDB
     HARVEST_GRAPHS_S3_FILENAME_PREFIX = ""  # Useful to store the graphs inside a subfolder of the bucket. For example by setting `HARVEST_GRAPHS_S3_FILENAME_PREFIX = 'graphs/'`
 
+    HARVEST_ISO19139_XSL_URL = "https://raw.githubusercontent.com/SEMICeu/iso-19139-to-dcat-ap/refs/heads/geodcat-ap-2.0.0/iso-19139-to-dcat-ap.xsl"
+
     # S3 connection details
     S3_URL = None
     S3_ACCESS_KEY_ID = None
@@ -508,6 +510,10 @@ class Defaults(object):
     ARCHIVE_COMMENT_USER_ID = None
     ARCHIVE_COMMENT_TITLE = _("This dataset has been archived")
 
+    # Transport extras
+    ##################
+    TRANSPORT_DATASETS_URL = None
+
     # Schemas parameters
     ####################
     SCHEMA_CATALOG_URL = None

udata/tasks.py

@@ -163,6 +163,7 @@ def init_app(app):
     import udata.core.tags.tasks  # noqa
     import udata.core.activity.tasks  # noqa
     import udata.core.dataset.tasks  # noqa
+    import udata.core.dataset.transport  # noqa
     import udata.core.dataset.recommendations  # noqa
     import udata.core.spatial.tasks  # noqa
     import udata.core.reuse.tasks  # noqa

udata/tests/dataset/test_transport_tasks.py

@@ -0,0 +1,75 @@
+import pytest
+import requests_mock
+
+from udata.core.dataset.factories import DatasetFactory
+from udata.core.dataset.transport import clear_datasets, map_transport_datasets
+
+
+@pytest.fixture
+def mock_response():
+    return [
+        {
+            "datagouv_id": "61fd29da29ea95c7bc0e1211",
+            "id": "61fd29da29ea95c7bc0e1211",
+            "page_url": "https://transport.data.gouv.fr/datasets/horaires-theoriques-et-temps-reel-des-navettes-hivernales-de-lalpe-dhuez-gtfs-gtfs-rt",
+            "slug": "horaires-theoriques-et-temps-reel-des-navettes-hivernales-de-lalpe-dhuez-gtfs-gtfs-rt",
+            "title": "Navettes hivernales de l'Alpe d'Huez",
+        },
+        {
+            "datagouv_id": "5f23d4b3d39755210a04a99c",
+            "id": "5f23d4b3d39755210a04a99c",
+            "page_url": "https://transport.data.gouv.fr/datasets/horaires-theoriques-et-temps-reel-du-reseau-lr-11-lalouvesc-tournon-st-felicien-gtfs-gtfs-rt",
+            "slug": "horaires-theoriques-et-temps-reel-du-reseau-lr-11-lalouvesc-tournon-st-felicien-gtfs-gtfs-rt",
+            "title": "Réseau interurbain  Lalouvesc / Tournon / St Felicien",
+        },
+    ]
+
+
+@pytest.mark.usefixtures("clean_db")
+class TransportTasksTest:
+    @pytest.mark.options(TRANSPORT_DATASETS_URL="http://local.test/api/datasets")
+    def test_map_transport_datasets(self, mock_response):
+        ds1 = DatasetFactory(id="61fd29da29ea95c7bc0e1211")
+        ds2 = DatasetFactory(id="5f23d4b3d39755210a04a99c")
+
+        with requests_mock.Mocker() as m:
+            m.get("http://local.test/api/datasets", json=mock_response)
+            map_transport_datasets()
+
+        ds1.reload()
+        ds2.reload()
+
+        assert (
+            ds1.extras["transport:url"]
+            == "https://transport.data.gouv.fr/datasets/horaires-theoriques-et-temps-reel-des-navettes-hivernales-de-lalpe-dhuez-gtfs-gtfs-rt"
+        )
+        assert (
+            ds2.extras["transport:url"]
+            == "https://transport.data.gouv.fr/datasets/horaires-theoriques-et-temps-reel-du-reseau-lr-11-lalouvesc-tournon-st-felicien-gtfs-gtfs-rt"
+        )
+
+        clear_datasets()
+
+        ds1.reload()
+        ds2.reload()
+
+        assert "transport:url" not in ds1.extras
+        assert "transport:url" not in ds2.extras
+
+    @pytest.mark.options(TRANSPORT_DATASETS_URL="http://local.test/api/datasets")
+    def test_map_transport_datasets_fail(self, mock_response):
+        """
+        We should not erase existing transport:url extras if the job fails
+        """
+        ds1 = DatasetFactory(id="61fd29da29ea95c7bc0e1211", extras={"transport:url": "dummy"})
+        ds2 = DatasetFactory(id="5f23d4b3d39755210a04a99c")
+
+        with requests_mock.Mocker() as m:
+            m.get("http://local.test/api/datasets", status_code=500)
+            map_transport_datasets()
+
+        ds1.reload()
+        ds2.reload()
+
+        assert ds1.extras["transport:url"] == "dummy"
+        assert "transport:url" not in ds2.extras

{udata-10.5.1.dev36034.dist-info → udata-10.5.1.dev36076.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: udata
-Version: 10.5.1.dev36034
+Version: 10.5.1.dev36076
 Summary: Open data portal
 Home-page: https://github.com/opendatateam/udata
 Author: Opendata Team
@@ -142,12 +142,14 @@ It is collectively taken care of by members of the
 
 ## Current (in progress)
 
+- Add config entry for ISO-DCAT XSLT URL [#3360](https://github.com/opendatateam/udata/pull/3360)
 - Fix failing to return dataservice in harvest preview because of no ID for URL [#3357](https://github.com/opendatateam/udata/pull/3357/)
 - Fix invalid resource format from harvested RDF records [#3354](https://github.com/opendatateam/udata/pull/3354)
 - Expose `dataset_id` for CommunityResource in /dataset/resource/id [#3258](https://github.com/opendatateam/udata/pull/3258)
 - Add a CI job to create a release on Sentry [#3266](https://github.com/opendatateam/udata/pull/3266)
 - Sort extensions in `/api/1/datasets/extensions/` response alphabetically [#3358](https://github.com/opendatateam/udata/pull/3358)
 - Migrate recommendations tasks, [udata-recommendations](https://github.com/opendatateam/udata-recommendations/) is no longer required [#3355](https://github.com/opendatateam/udata/pull/3355)
+- Migrate udata-transport tasks, [udata-transport](https://github.com/opendatateam/udata-transport/) is no longer required [#3355](https://github.com/opendatateam/udata/pull/3355)
 
 ## 10.5.0 (2025-07-02)
 

{udata-10.5.1.dev36034.dist-info → udata-10.5.1.dev36076.dist-info}/RECORD

@@ -13,10 +13,10 @@ udata/mail.py,sha256=FMGHcDAjHvk86iDUwBmVXpx3vbAb2c-j5C3BRnh9IYQ,2670
 udata/rdf.py,sha256=JmMxwq4fFBrBZQhJ6O9_nEeYUXspPzoZGTyGUD4Nyxs,18348
 udata/routing.py,sha256=E6sE1F74QyOoz5vcgEi-rNEhCegwLfOtBz5I9fWk-pM,7677
 udata/sentry.py,sha256=ekcxqUSqxfM98TtvCsPaOoX5i2l6PEcYt7kb4l3od-Q,3223
-udata/settings.py,sha256=jXflM02A-l1qeGcjegRtkw56cbQUDOPzqY85OBs_dV0,19345
+udata/settings.py,sha256=DPG8V4Qkqc3jv1evxvEOlxyZH6h0wphKcP-b5D-d8D4,19576
 udata/sitemap.py,sha256=oRRWoPI7ZsFFnUAOqGT1YuXFFKHBe8EcRnUCNHD7xjM,979
 udata/tags.py,sha256=ydq4uokd6bzdeGVSpEXASVtGvDfO2LfQs9mptvvKJCM,631
-udata/tasks.py,sha256=Tt2Vy4fWyBkDz6cK4Mx3g2VdOLgfdDkWSNIc1QpoawQ,5032
+udata/tasks.py,sha256=yTYBJG5bzEChX27p3MSqurSji84rg7w7OUvK4vuPRfY,5080
 udata/terms.md,sha256=nFx978tUQ3vTEv6POykXaZvcQ5e_gcvmO4ZgcfbSWXo,187
 udata/tracking.py,sha256=WOcqA1RlHN8EPFuEc2kNau54mec4-pvi-wUFrMXevzg,345
 udata/uris.py,sha256=sIhlzpwFO7ftOHYgTZmR7mCoty6a1n4KC4c0Qmx3lqo,3655
@@ -113,6 +113,7 @@ udata/core/dataset/recommendations.py,sha256=DlGSLU8D0nW6Ds1rjBav1WxC-0VW5yOCjkO
 udata/core/dataset/search.py,sha256=E7LqHBnq3sMefvmLwTpiw-Ovem2a3NJswHesRjctboE,5627
 udata/core/dataset/signals.py,sha256=WN4sV-lJlNsRkhcnhoy0SYJvCoYmK_5QFYZd1u-h4gs,161
 udata/core/dataset/tasks.py,sha256=6FzeLzJRQxzq7sBLUE8H8ZGLByix2EDOzGAsA8FteX8,10019
+udata/core/dataset/transport.py,sha256=ihCXirY1dZjOfXKbf9HRCJTfIOc75rM1McwbeGjsW6A,1296
 udata/core/discussions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 udata/core/discussions/actions.py,sha256=kjdBLDIeu0yWTSxQGgOpN1WoxUqbMygn4SiBk_S9T5I,1051
 udata/core/discussions/api.py,sha256=u9atdyPVGpyLin4IRF1ZhMp6gRmtJvV0Y8ksic_nbJo,12807
@@ -297,7 +298,7 @@ udata/harvest/signals.py,sha256=3AhFHMPIFH5vz01NX5ycR_RWH14MXFWnCT6__LSa-QI,1338
 udata/harvest/tasks.py,sha256=ddJtvE0s-kAYt27-rKH6n8U8vKD8qKczlJtdBJzMUns,1718
 udata/harvest/backends/__init__.py,sha256=QjoFfBJfpw_xgk5YYWI1SgKJOMEmTMlxSfW79GNkSTI,459
 udata/harvest/backends/base.py,sha256=2wyfw83e3xGQcHnQI-z26g1dg-uVtWcDgzsBk7iGX3Y,17480
-udata/harvest/backends/dcat.py,sha256=ofd3JiKL4x02GAPKtW1wryoZD0_P4xnwoZC2KWp_sNI,18698
+udata/harvest/backends/dcat.py,sha256=IfUmqHy68Tt30E_kn4riExfgb-d7qPa--7KjCLCsia0,18637
 udata/harvest/backends/maaf.py,sha256=N7ty8ZWO9pfKPtZRk1wTaJ5pY6qi-0-GtF1p8jiYiY4,8102
 udata/harvest/backends/maaf.xsd,sha256=vEyG8Vqw7Yn_acjRdXjqUJgxOj4pv8bibep-FX-f3BQ,18322
 udata/harvest/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -306,7 +307,7 @@ udata/harvest/tests/person.jsonld,sha256=I7Ynh-PQlNeD51I1LrCgYOEjhL-WBeb65xzIE_s
 udata/harvest/tests/test_actions.py,sha256=Hm5MpOEFXBvFzcNMu7hvrWIiP6fiOXLuDOyrykL-4tc,25224
 udata/harvest/tests/test_api.py,sha256=dyCXhKgOO0XmCPgH_36sob8WVhtMB2WX1johbfzJtDI,21553
 udata/harvest/tests/test_base_backend.py,sha256=ow8ecGtD836mUqyPWYjkS5nx0STyT5RMLgBdDyOhts4,19233
-udata/harvest/tests/test_dcat_backend.py,sha256=O2SOSz34DwBxta9RrDJ2wGxxfwTKFyCHTEc1bH0oRI8,43717
+udata/harvest/tests/test_dcat_backend.py,sha256=DCRtA0DDW7XrX7ux7GI6HaUcI-ZOWIoGe8Dl2pwGCo4,43713
 udata/harvest/tests/test_filters.py,sha256=PT2qopEIoXsqi8MsNDRuhNH7jGXiQo8r0uJrCOUd4aM,2465
 udata/harvest/tests/test_models.py,sha256=f9NRR2_S4oZFgF8qOumg0vv-lpnEBJbI5vNtcwFdSqM,831
 udata/harvest/tests/test_notifications.py,sha256=MMzTzkv-GXMNFeOwAi31rdTsAXyLCLOSna41zOtaJG0,816
@@ -669,6 +670,7 @@ udata/tests/dataset/test_dataset_rdf.py,sha256=7SorX0e0VD3hmj8C0qXA4Vb3Q3xl2qaE4
 udata/tests/dataset/test_dataset_recommendations.py,sha256=K52HXGXi9DuUSiSRQWpqTrsFCkYbv7K3upNp0fFp5v8,7068
 udata/tests/dataset/test_dataset_tasks.py,sha256=n1W2Pg0ez02d66zQG3N93kh7dpR2yLMRDqUI6PnPaI0,3088
 udata/tests/dataset/test_resource_preview.py,sha256=fp9mSL7unhyM66GR0gwhgX3OGQ4TJt7G9xU-CjsL3HI,3908
+udata/tests/dataset/test_transport_tasks.py,sha256=BYr1WPV0Crirzb2jC_wAV4y_pzImRCAS4zYZ7qvCDeY,2964
 udata/tests/features/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 udata/tests/features/territories/__init__.py,sha256=gMD73RL-ymcWvGPDPM0aPxz7WAfd1VEDL8YHRI7HT0Q,956
 udata/tests/features/territories/test_territories_api.py,sha256=UA5j5ZqLP3L3PdddMYJdizmftUFIjj3wWHOt5M2kQtU,7527
@@ -734,9 +736,9 @@ udata/translations/pt/LC_MESSAGES/udata.mo,sha256=9sCd1MUKvtVP_sOXvK-G5v4PfWkkdA
 udata/translations/pt/LC_MESSAGES/udata.po,sha256=-eJptz9s63rjkdm-3HJi_2t70pyv3-8EuXBn-B2qI_4,48419
 udata/translations/sr/LC_MESSAGES/udata.mo,sha256=qduXntHWe__KaUxJ4JwwyGG3eSgYb1auGdNax0lS49c,29169
 udata/translations/sr/LC_MESSAGES/udata.po,sha256=6QCuLMCRjgyAvu9U7i0P19ae8fm_uStfmxHLqUO9EoY,55394
-udata-10.5.1.dev36034.dist-info/LICENSE,sha256=V8j_M8nAz8PvAOZQocyRDX7keai8UJ9skgmnwqETmdY,34520
-udata-10.5.1.dev36034.dist-info/METADATA,sha256=j-kbOdj5x8dcfnIMgvlLhf-_h2pEL5-O9XNwNL0lANA,149656
-udata-10.5.1.dev36034.dist-info/WHEEL,sha256=Kh9pAotZVRFj97E15yTA4iADqXdQfIVTHcNaZTjxeGM,110
-udata-10.5.1.dev36034.dist-info/entry_points.txt,sha256=ETvkR4r6G1duBsh_V_fGWENQy17GTFuobi95MYBAl1A,498
-udata-10.5.1.dev36034.dist-info/top_level.txt,sha256=39OCg-VWFWOq4gCKnjKNu-s3OwFlZIu_dVH8Gl6ndHw,12
-udata-10.5.1.dev36034.dist-info/RECORD,,
+udata-10.5.1.dev36076.dist-info/LICENSE,sha256=V8j_M8nAz8PvAOZQocyRDX7keai8UJ9skgmnwqETmdY,34520
+udata-10.5.1.dev36076.dist-info/METADATA,sha256=7diG5zHyeuMPGlb1wYgvigaapy8Di8PQ_JYTRP4Eb5c,149934
+udata-10.5.1.dev36076.dist-info/WHEEL,sha256=Kh9pAotZVRFj97E15yTA4iADqXdQfIVTHcNaZTjxeGM,110
+udata-10.5.1.dev36076.dist-info/entry_points.txt,sha256=ETvkR4r6G1duBsh_V_fGWENQy17GTFuobi95MYBAl1A,498
+udata-10.5.1.dev36076.dist-info/top_level.txt,sha256=39OCg-VWFWOq4gCKnjKNu-s3OwFlZIu_dVH8Gl6ndHw,12
+udata-10.5.1.dev36076.dist-info/RECORD,,