udata 10.1.2.dev34172__py2.py3-none-any.whl → 10.1.3__py2.py3-none-any.whl

udata/harvest/actions.py

@@ -7,6 +7,7 @@ from bson import ObjectId
 from flask import current_app
 
 from udata.auth import current_user
+from udata.core.dataservices.models import Dataservice
 from udata.core.dataset.models import HarvestDatasetMetadata
 from udata.models import Dataset, Organization, PeriodicTask, User
 from udata.storage.s3 import delete_file
@@ -18,6 +19,7 @@ from .models import (
     VALIDATION_REFUSED,
     HarvestJob,
     HarvestSource,
+    archive_harvested_dataservice,
     archive_harvested_dataset,
 )
 from .tasks import harvest
@@ -161,6 +163,9 @@ def purge_sources():
         datasets = Dataset.objects.filter(harvest__source_id=str(source.id))
         for dataset in datasets:
             archive_harvested_dataset(dataset, reason="harvester-deleted", dryrun=False)
+        dataservices = Dataservice.objects.filter(harvest__source_id=str(source.id))
+        for dataservice in dataservices:
+            archive_harvested_dataservice(dataservice, reason="harvester-deleted", dryrun=False)
         source.delete()
     return count
 

udata/harvest/backends/base.py

@@ -20,6 +20,7 @@ from ..models import (
     HarvestItem,
     HarvestJob,
     HarvestLog,
+    archive_harvested_dataservice,
     archive_harvested_dataset,
 )
 from ..signals import after_harvest_job, before_harvest_job
@@ -342,6 +343,7 @@ class BaseBackend(object):
         harvest.last_update = datetime.utcnow()
 
         harvest.archived_at = None
+        harvest.archived_reason = None
 
         return harvest
 
@@ -370,9 +372,10 @@ class BaseBackend(object):
             "harvest__remote_id__nin": remote_ids,
             "harvest__last_update__lt": limit_date,
         }
-        local_items_not_on_remote = Dataset.objects.filter(**q)
+        local_datasets_not_on_remote = Dataset.objects.filter(**q)
+        local_dataservices_not_on_remote = Dataservice.objects.filter(**q)
 
-        for dataset in local_items_not_on_remote:
+        for dataset in local_datasets_not_on_remote:
             if not dataset.harvest.archived_at:
                 archive_harvested_dataset(dataset, reason="not-on-remote", dryrun=self.dryrun)
             # add a HarvestItem to the job list (useful for report)
@@ -385,6 +388,23 @@ class BaseBackend(object):
 
             self.save_job()
 
+        for dataservice in local_dataservices_not_on_remote:
+            if not dataservice.harvest.archived_at:
+                archive_harvested_dataservice(
+                    dataservice, reason="not-on-remote", dryrun=self.dryrun
+                )
+            # add a HarvestItem to the job list (useful for report)
+            # even when archiving has already been done (useful for debug)
+            self.job.items.append(
+                HarvestItem(
+                    remote_id=str(dataservice.harvest.remote_id),
+                    dataservice=dataservice,
+                    status="archived",
+                )
+            )
+
+            self.save_job()
+
     def get_dataset(self, remote_id):
         """Get or create a dataset given its remote ID (and its source)
         We first try to match `source_id` to be source domain independent

udata/harvest/models.py

@@ -6,6 +6,7 @@ from urllib.parse import urlparse
 from werkzeug.utils import cached_property
 
 from udata.core.dataservices.models import Dataservice
+from udata.core.dataservices.models import HarvestMetadata as HarvestDataserviceMetadata
 from udata.core.dataset.models import HarvestDatasetMetadata
 from udata.core.owned import Owned, OwnedQuerySet
 from udata.i18n import lazy_gettext as _
@@ -203,3 +204,21 @@ def archive_harvested_dataset(dataset, reason, dryrun=False):
         dataset.validate()
     else:
         dataset.save()
+
+
+def archive_harvested_dataservice(dataservice, reason, dryrun=False):
+    """
+    Archive an harvested dataservice, setting extras accordingly.
+    If `dryrun` is True, the dataservice is not saved but validated only.
+    """
+    log.debug("Archiving dataservice %s", dataservice.id)
+    archival_date = datetime.utcnow()
+    dataservice.archived_at = archival_date
+    if not dataservice.harvest:
+        dataservice.harvest = HarvestDataserviceMetadata()
+    dataservice.harvest.archived_reason = reason
+    dataservice.harvest.archived_at = archival_date
+    if dryrun:
+        dataservice.validate()
+    else:
+        dataservice.save()

udata/harvest/tests/test_actions.py

@@ -6,6 +6,8 @@ from tempfile import NamedTemporaryFile
 import pytest
 from mock import patch
 
+from udata.core.dataservices.factories import DataserviceFactory
+from udata.core.dataservices.models import HarvestMetadata as HarvestDataserviceMetadata
 from udata.core.dataset.factories import DatasetFactory
 from udata.core.dataset.models import HarvestDatasetMetadata
 from udata.core.organization.factories import OrganizationFactory
@@ -396,17 +398,27 @@ class HarvestActionsTest:
         dataset_to_archive = DatasetFactory(
             harvest=HarvestDatasetMetadata(source_id=str(to_delete[0].id))
         )
+        dataservice_to_archive = DataserviceFactory(
+            harvest=HarvestDataserviceMetadata(source_id=str(to_delete[0].id))
+        )
 
         result = actions.purge_sources()
         dataset_to_archive.reload()
+        dataservice_to_archive.reload()
 
         assert result == len(to_delete)
         assert len(HarvestSource.objects) == len(to_keep)
         assert PeriodicTask.objects.filter(id=periodic_task.id).count() == 0
         assert HarvestJob.objects(id=harvest_job.id).count() == 0
+
         assert dataset_to_archive.harvest.archived == "harvester-deleted"
+        assert_equal_dates(dataset_to_archive.harvest.archived_at, now)
         assert_equal_dates(dataset_to_archive.archived, now)
 
+        assert dataservice_to_archive.harvest.archived_reason == "harvester-deleted"
+        assert_equal_dates(dataservice_to_archive.harvest.archived_at, now)
+        assert_equal_dates(dataservice_to_archive.archived_at, now)
+
     @pytest.mark.options(HARVEST_JOBS_RETENTION_DAYS=2)
     def test_purge_jobs(self):
         now = datetime.utcnow()

udata/harvest/tests/test_base_backend.py

@@ -4,6 +4,8 @@ from urllib.parse import urlparse
 import pytest
 from voluptuous import Schema
 
+from udata.core.dataservices.factories import DataserviceFactory
+from udata.core.dataservices.models import Dataservice
 from udata.core.dataset import tasks
 from udata.core.dataset.factories import DatasetFactory
 from udata.harvest.models import HarvestItem
@@ -20,9 +22,9 @@ class Unknown:
     pass
 
 
-def gen_remote_IDs(num: int) -> list[str]:
+def gen_remote_IDs(num: int, prefix: str = "") -> list[str]:
     """Generate remote IDs."""
-    return [f"fake-{i}" for i in range(num)]
+    return [f"{prefix}fake-{i}" for i in range(num)]
 
 
 class FakeBackend(BaseBackend):
@@ -45,6 +47,11 @@ class FakeBackend(BaseBackend):
             if self.is_done():
                 return
 
+        for remote_id in self.source.config.get("dataservice_remote_ids", []):
+            self.process_dataservice(remote_id)
+            if self.is_done():
+                return
+
     def inner_process_dataset(self, item: HarvestItem):
         dataset = self.get_dataset(item.remote_id)
 
@@ -55,6 +62,16 @@ class FakeBackend(BaseBackend):
             dataset.last_modified_internal = self.source.config["last_modified"]
         return dataset
 
+    def inner_process_dataservice(self, item: HarvestItem):
+        dataservice = self.get_dataservice(item.remote_id)
+
+        for key, value in DataserviceFactory.as_dict().items():
+            if getattr(dataservice, key) is None:
+                setattr(dataservice, key, value)
+        if self.source.config.get("last_modified"):
+            dataservice.last_modified_internal = self.source.config["last_modified"]
+        return dataservice
+
 
 class HarvestFilterTest:
     @pytest.mark.parametrize("type,expected", HarvestFilter.TYPES.items())
@@ -210,7 +227,13 @@ class BaseBackendTest:
 
     def test_autoarchive(self, app):
         nb_datasets = 3
-        source = HarvestSourceFactory(config={"dataset_remote_ids": gen_remote_IDs(nb_datasets)})
+        nb_dataservices = 3
+        source = HarvestSourceFactory(
+            config={
+                "dataset_remote_ids": gen_remote_IDs(nb_datasets, "dataset-"),
+                "dataservice_remote_ids": gen_remote_IDs(nb_dataservices, "dataservice-"),
+            }
+        )
         backend = FakeBackend(source)
 
         # create a dangling dataset to be archived
@@ -220,7 +243,15 @@ class BaseBackendTest:
             harvest={
                 "domain": source.domain,
                 "source_id": str(source.id),
-                "remote_id": "not-on-remote",
+                "remote_id": "dataset-not-on-remote",
+                "last_update": last_update,
+            }
+        )
+        dataservice_arch = DataserviceFactory(
+            harvest={
+                "domain": source.domain,
+                "source_id": str(source.id),
+                "remote_id": "dataservice-not-on-remote",
                 "last_update": last_update,
             }
         )
@@ -232,7 +263,15 @@ class BaseBackendTest:
             harvest={
                 "domain": source.domain,
                 "source_id": str(source.id),
-                "remote_id": "not-on-remote-two",
+                "remote_id": "dataset-not-on-remote-two",
+                "last_update": last_update,
+            }
+        )
+        dataservice_no_arch = DataserviceFactory(
+            harvest={
+                "domain": source.domain,
+                "source_id": str(source.id),
+                "remote_id": "dataservice-not-on-remote-two",
                 "last_update": last_update,
             }
         )
@@ -240,13 +279,17 @@ class BaseBackendTest:
         job = backend.harvest()
 
         # all datasets except arch : 3 mocks + 1 manual (no_arch)
-        assert len(job.items) == nb_datasets + 1
+        assert len(job.items) == (nb_datasets + 1) + (nb_dataservices + 1)
         # all datasets : 3 mocks + 2 manuals (arch and no_arch)
         assert Dataset.objects.count() == nb_datasets + 2
+        assert Dataservice.objects.count() == nb_dataservices + 2
 
         archived_items = [i for i in job.items if i.status == "archived"]
-        assert len(archived_items) == 1
+        assert len(archived_items) == 2
         assert archived_items[0].dataset == dataset_arch
+        assert archived_items[0].dataservice is None
+        assert archived_items[1].dataset is None
+        assert archived_items[1].dataservice == dataservice_arch
 
         dataset_arch.reload()
         assert dataset_arch.archived is not None
@@ -258,18 +301,41 @@ class BaseBackendTest:
         assert "archived" not in dataset_no_arch.harvest
         assert "archived_at" not in dataset_no_arch.harvest
 
+        dataservice_arch.reload()
+        assert dataservice_arch.archived_at is not None
+        assert "archived_reason" in dataservice_arch.harvest
+        assert "archived_at" in dataservice_arch.harvest
+
+        dataservice_no_arch.reload()
+        assert dataservice_no_arch.archived_at is None
+        assert "archived_reason" not in dataservice_no_arch.harvest
+        assert "archived_at" not in dataservice_no_arch.harvest
+
         # test unarchive: archive manually then relaunch harvest
-        dataset = Dataset.objects.get(**{"harvest__remote_id": "fake-1"})
+        dataset = Dataset.objects.get(**{"harvest__remote_id": "dataset-fake-1"})
         dataset.archived = datetime.utcnow()
         dataset.harvest.archived = "not-on-remote"
         dataset.harvest.archived_at = datetime.utcnow()
         dataset.save()
+
+        dataservice = Dataservice.objects.get(**{"harvest__remote_id": "dataservice-fake-1"})
+        dataservice.archived_at = datetime.utcnow()
+        dataservice.harvest.archived_reason = "not-on-remote"
+        dataservice.harvest.archived_at = datetime.utcnow()
+        dataservice.save()
+
         backend.harvest()
+
         dataset.reload()
         assert dataset.archived is None
         assert "archived" not in dataset.harvest
         assert "archived_at" not in dataset.harvest
 
+        dataservice.reload()
+        assert dataservice.archived_at is None
+        assert "archived_reason" not in dataservice.harvest
+        assert "archived_at" not in dataservice.harvest
+
     def test_harvest_datasets_get_deleted(self):
         nb_datasets = 3
         source = HarvestSourceFactory(config={"dataset_remote_ids": gen_remote_IDs(nb_datasets)})

udata/harvest/tests/test_dcat_backend.py

@@ -179,7 +179,7 @@ class DcatBackendTest:
         assert dataservices[0].title == "Explore API v2"
         assert dataservices[0].base_api_url == "https://data.paris2024.org/api/explore/v2.1/"
         assert (
-            dataservices[0].endpoint_description_url
+            dataservices[0].machine_documentation_url
             == "https://data.paris2024.org/api/explore/v2.1/swagger.json"
         )
         assert (

udata/migrations/2025-01-05-dataservices-fields-changes.py

@@ -0,0 +1,136 @@
+"""
+This migration keeps only the "Local authority" badge if the organization also has the "Public service" badge.
+"""
+
+import logging
+from typing import List
+
+from mongoengine.connection import get_db
+
+from udata.core.dataservices.constants import (
+    DATASERVICE_ACCESS_TYPE_OPEN,
+    DATASERVICE_ACCESS_TYPE_OPEN_WITH_ACCOUNT,
+    DATASERVICE_ACCESS_TYPE_RESTRICTED,
+)
+from udata.core.dataservices.models import Dataservice
+
+log = logging.getLogger(__name__)
+
+
+def migrate(db):
+    log.info("Preprocessing dataservices…")
+
+    count = get_db().dataservice.update_many(
+        filter={
+            "$or": [
+                {"is_restricted": None},
+                {"is_restricted": {"$exists": False}},
+            ]
+        },
+        update={"$set": {"is_restricted": False}},
+    )
+    log.info(
+        f"\tConverted {count.modified_count} dataservices from `is_restricted=None` to `is_restricted=False`"
+    )
+
+    count = get_db().dataservice.update_many(
+        filter={
+            "$or": [
+                {"has_token": None},
+                {"has_token": {"$exists": False}},
+            ]
+        },
+        update={"$set": {"has_token": False}},
+    )
+    log.info(
+        f"\tConverted {count.modified_count} dataservices from `has_token=None` to `has_token=False`"
+    )
+
+    for dataservice in get_db().dataservice.find({"is_restricted": True, "has_token": False}):
+        log.info(
+            f"\tDataservice #{dataservice['_id']} {dataservice['title']} is restricted but without token. (will be set to access_type={DATASERVICE_ACCESS_TYPE_RESTRICTED})"
+        )
+
+    log.info("Processing dataservices…")
+
+    count = get_db().dataservice.update_many(
+        filter={
+            "is_restricted": True,
+            # `has_token` could be True or False, we don't care
+        },
+        update={"$set": {"access_type": DATASERVICE_ACCESS_TYPE_RESTRICTED}},
+    )
+    log.info(
+        f"\t{count.modified_count} restricted dataservices to DATASERVICE_ACCESS_TYPE_RESTRICTED"
+    )
+
+    count = get_db().dataservice.update_many(
+        filter={
+            "is_restricted": False,
+            "has_token": True,
+        },
+        update={"$set": {"access_type": DATASERVICE_ACCESS_TYPE_OPEN_WITH_ACCOUNT}},
+    )
+    log.info(
+        f"\t{count.modified_count} dataservices not restricted but with token to DATASERVICE_ACCESS_TYPE_OPEN_WITH_ACCOUNT"
+    )
+
+    count = get_db().dataservice.update_many(
+        filter={
+            "is_restricted": False,
+            "has_token": False,
+        },
+        update={"$set": {"access_type": DATASERVICE_ACCESS_TYPE_OPEN}},
+    )
+    log.info(f"\t{count.modified_count} open dataservices to DATASERVICE_ACCESS_TYPE_OPEN")
+
+    dataservices: List[Dataservice] = get_db().dataservice.find()
+    for dataservice in dataservices:
+        if (
+            "endpoint_description_url" not in dataservice
+            or not dataservice["endpoint_description_url"]
+        ):
+            continue
+
+        to_set = {}
+        if (
+            dataservice["endpoint_description_url"].endswith(".json")
+            or dataservice["endpoint_description_url"].endswith(".yaml")
+            or dataservice["endpoint_description_url"].endswith(".yml")
+            or dataservice["endpoint_description_url"].endswith("?format=openapi-json")
+            or "getcapabilities" in dataservice["endpoint_description_url"].lower()
+            or "getresourcedescription" in dataservice["endpoint_description_url"].lower()
+            or dataservice["endpoint_description_url"].startswith(
+                "https://api.insee.fr/catalogue/api-docs/carbon.super"
+            )
+        ):
+            # log.info(f"[MACHINE] {dataservice["endpoint_description_url"]}")
+            to_set["machine_documentation_url"] = dataservice["endpoint_description_url"]
+        else:
+            # log.info(f"[ HUMAN ] {dataservice["endpoint_description_url"]}")
+            to_set["technical_documentation_url"] = dataservice["endpoint_description_url"]
+
+        result = get_db().dataservice.update_one(
+            filter={
+                "_id": dataservice["_id"],
+            },
+            update={"$set": to_set},
+        )
+        assert result.modified_count == 1
+        assert result.matched_count == 1
+
+    log.info("Postprocessing dataservices…")
+
+    count = get_db().dataservice.update_many(
+        {},
+        {
+            "$unset": {
+                "endpoint_description_url": "",
+                "is_restricted": "",
+                "has_token": "",
+            }
+        },
+    )
+    log.info(f"\tUnset legacy fields on {count.modified_count} dataservices")
+
+    log.info("Done")

udata/settings.py

@@ -115,6 +115,11 @@ class Defaults(object):
 
     SECURITY_RETURN_GENERIC_RESPONSES = False
 
+    # Inactive users settings
+    YEARS_OF_INACTIVITY_BEFORE_DELETION = None
+    DAYS_BEFORE_ACCOUNT_INACTIVITY_NOTIFY_DELAY = 30
+    MAX_NUMBER_OF_USER_INACTIVITY_NOTIFICATIONS = 200
+
     # Sentry configuration
     SENTRY_DSN = None
     SENTRY_TAGS = {}

udata/templates/mail/account_inactivity.html

@@ -0,0 +1,29 @@
+{% extends 'mail/base.html' %}
+{% from 'mail/button.html' import mail_button %}
+
+{% block body %}
+<p style="margin: 0;padding: 0;">
+    {{ _(
+            'Your account  (%(user_email)s) has been inactive for %(inactivity_years)d years or more.',
+            user_email=user.email,
+            inactivity_years=config.YEARS_OF_INACTIVITY_BEFORE_DEACTIVATION
+        )
+    }}
+</p>
+<br/>
+<p style="margin: 0;padding: 0;"><b>
+    {{ _(
+        'If you want to keep your account, please log in with your account on %(site)s.',
+        site=config.SITE_TITLE
+        )
+    }}
+</b></p>
+<br/>
+<p style="margin: 0;padding: 0;">
+    {{ _(
+        'Without logging in, your account will be deleted within %(notify_delay)d days.',
+        notify_delay=config.DAYS_BEFORE_ACCOUNT_INACTIVITY_NOTIFY_DELAY
+        )
+    }}
+</p>
+{% endblock %}

udata/templates/mail/account_inactivity.txt

@@ -0,0 +1,22 @@
+{% extends 'mail/base.txt' %}
+
+{% block body %}
+{{ _(
+      'Your account  (%(user_email)s) has been inactive for %(inactivity_years)d years or more.',
+      user_email=user.email,
+      inactivity_years=config.YEARS_OF_INACTIVITY_BEFORE_DEACTIVATION
+    )
+}}
+
+{{ _(
+      'If you want to keep your account, please log in with your account on %(site)s.',
+      site=config.SITE_TITLE
+    )
+}}
+
+{{ _(
+      'Without logging in, your account will be deleted within %(notify_delay)d days.',
+      notify_delay=config.DAYS_BEFORE_ACCOUNT_INACTIVITY_NOTIFY_DELAY
+    )
+}}
+{% endblock %}

udata/templates/mail/inactive_account_deleted.html

@@ -0,0 +1,5 @@
+{% extends 'mail/base.html' %}
+
+{% block body %}
+<p style="margin: 0;padding: 0;">{{ _('Your account on %(site)s has been deleted due to inactivity', site=config.SITE_TITLE) }}</p>
+{% endblock %}

udata/templates/mail/inactive_account_deleted.txt

@@ -0,0 +1,6 @@
+{% extends 'mail/base.txt' %}
+
+{% block body %}
+{{ _('Your account on %(site)s has been deleted due to inactivity', site=config.SITE_TITLE) }}.
+
+{% endblock %}

udata/tests/api/test_dataservices_api.py

@@ -5,6 +5,11 @@ from flask import url_for
 from werkzeug.test import TestResponse
 
 import udata.core.organization.constants as org_constants
+from udata.core.dataservices.constants import (
+    DATASERVICE_ACCESS_TYPE_OPEN,
+    DATASERVICE_ACCESS_TYPE_OPEN_WITH_ACCOUNT,
+    DATASERVICE_ACCESS_TYPE_RESTRICTED,
+)
 from udata.core.dataservices.factories import DataserviceFactory
 from udata.core.dataservices.models import Dataservice
 from udata.core.dataset.factories import DatasetFactory, LicenseFactory
@@ -246,6 +251,7 @@ class DataserviceAPITest(APITestCase):
                 "title": "B",
                 "base_api_url": "https://example.org/B",
                 "datasets": [dataset_b.id],
+                "access_type": DATASERVICE_ACCESS_TYPE_OPEN,
             },
         )
         self.post(
@@ -254,6 +260,7 @@ class DataserviceAPITest(APITestCase):
                 "title": "C",
                 "base_api_url": "https://example.org/C",
                 "datasets": [dataset_a.id, dataset_b.id],
+                "access_type": DATASERVICE_ACCESS_TYPE_OPEN_WITH_ACCOUNT,
             },
         )
         self.post(
@@ -262,6 +269,7 @@ class DataserviceAPITest(APITestCase):
                 "title": "A",
                 "base_api_url": "https://example.org/A",
                 "datasets": [dataset_a.id],
+                "access_type": DATASERVICE_ACCESS_TYPE_RESTRICTED,
             },
         )
         self.post(
@@ -324,10 +332,27 @@ class DataserviceAPITest(APITestCase):
         self.assertEqual(response.json["data"][0]["title"], "A")
         self.assertEqual(response.json["data"][1]["title"], "C")
 
+        response = self.get(url_for("api.dataservices", access_type=DATASERVICE_ACCESS_TYPE_OPEN))
+        self.assert200(response)
+
+        print(response.json)
+        self.assertEqual(response.json["total"], 1)
+        self.assertEqual(response.json["data"][0]["title"], "B")
+
     def test_dataservice_api_index_with_sorts(self):
         DataserviceFactory(title="A", created_at="2024-03-01", metadata_modified_at="2024-03-01")
-        DataserviceFactory(title="B", created_at="2024-02-01", metadata_modified_at="2024-05-01")
-        DataserviceFactory(title="C", created_at="2024-05-01", metadata_modified_at="2024-04-01")
+        DataserviceFactory(
+            title="B",
+            created_at="2024-02-01",
+            metadata_modified_at="2024-05-01",
+            metrics={"views": 42},
+        )
+        DataserviceFactory(
+            title="C",
+            created_at="2024-05-01",
+            metadata_modified_at="2024-04-01",
+            metrics={"views": 1337},
+        )
         DataserviceFactory(title="D", created_at="2024-04-01", metadata_modified_at="2024-02-01")
 
         response = self.get(url_for("api.dataservices", sort="title"))
@@ -366,6 +391,20 @@ class DataserviceAPITest(APITestCase):
             [dataservice["title"] for dataservice in response.json["data"]], ["B", "C", "A", "D"]
         )
 
+        response = self.get(url_for("api.dataservices", sort="views"))
+        self.assert200(response)
+
+        self.assertEqual(
+            [dataservice["title"] for dataservice in response.json["data"]], ["A", "D", "B", "C"]
+        )
+
+        response = self.get(url_for("api.dataservices", sort="-views"))
+        self.assert200(response)
+
+        self.assertEqual(
+            [dataservice["title"] for dataservice in response.json["data"]], ["C", "B", "D", "A"]
+        )
+
     def test_dataservice_api_index_with_wrong_dataset_id(self):
         response = self.get(url_for("api.dataservices", sort="title", dataset=str("xxx")))
         self.assert400(response)