udata 10.1.3.dev34275__py2.py3-none-any.whl → 10.1.3.dev34303__py2.py3-none-any.whl

udata/core/dataservices/models.py

@@ -94,6 +94,7 @@ class HarvestMetadata(db.EmbeddedDocument):
     )
     last_update = field(db.DateTimeField(), description="Date of the last harvesting")
     archived_at = field(db.DateTimeField())
+    archived_reason = field(db.StringField())
 
 
 @generate_fields(

udata/core/user/api.py

@@ -275,6 +275,13 @@ delete_parser.add_argument(
     location="args",
     default=False,
 )
+delete_parser.add_argument(
+    "delete_comments",
+    type=bool,
+    help="Delete comments posted by the user upon user deletion",
+    location="args",
+    default=False,
+)
 
 
 @ns.route("/<user:user>/", endpoint="user")
@@ -317,7 +324,7 @@ class UserAPI(API):
                 403, "You cannot delete yourself with this API. " + 'Use the "me" API instead.'
             )
 
-        user.mark_as_deleted(notify=not args["no_mail"])
+        user.mark_as_deleted(notify=not args["no_mail"], delete_comments=args["delete_comments"])
         return "", 204
 
 

udata/core/user/models.py

@@ -82,6 +82,10 @@ class User(WithMetrics, UserMixin, db.Document):
     ext = db.MapField(db.GenericEmbeddedDocumentField())
     extras = db.ExtrasField()
 
+    # Used to track notification for automatic inactive users deletion
+    # when YEARS_OF_INACTIVITY_BEFORE_DEACTIVATION is set
+    inactive_deletion_notified_at = db.DateTimeField()
+
     before_save = Signal()
     after_save = Signal()
     on_create = Signal()
@@ -237,7 +241,7 @@ class User(WithMetrics, UserMixin, db.Document):
         raise NotImplementedError("""This method should not be using directly.
         Use `mark_as_deleted` (or `_delete` if you know what you're doing)""")
 
-    def mark_as_deleted(self, notify: bool = True):
+    def mark_as_deleted(self, notify: bool = True, delete_comments: bool = False):
         if self.avatar.filename is not None:
             storage = storages.avatars
             storage.delete(self.avatar.filename)
@@ -265,16 +269,17 @@ class User(WithMetrics, UserMixin, db.Document):
                 member for member in organization.members if member.user != self
             ]
             organization.save()
-        for discussion in Discussion.objects(discussion__posted_by=self):
-            # Remove all discussions with current user as only participant
-            if all(message.posted_by == self for message in discussion.discussion):
-                discussion.delete()
-                continue
-
-            for message in discussion.discussion:
-                if message.posted_by == self:
-                    message.content = "DELETED"
-            discussion.save()
+        if delete_comments:
+            for discussion in Discussion.objects(discussion__posted_by=self):
+                # Remove all discussions with current user as only participant
+                if all(message.posted_by == self for message in discussion.discussion):
+                    discussion.delete()
+                    continue
+
+                for message in discussion.discussion:
+                    if message.posted_by == self:
+                        message.content = "DELETED"
+                discussion.save()
         Follow.objects(follower=self).delete()
         Follow.objects(following=self).delete()
 

udata/core/user/tasks.py

@@ -1,10 +1,14 @@
 import logging
+from copy import copy
+from datetime import datetime, timedelta
+
+from flask import current_app
 
 from udata import mail
 from udata.i18n import lazy_gettext as _
-from udata.tasks import task
+from udata.tasks import job, task
 
-from .models import datastore
+from .models import User, datastore
 
 log = logging.getLogger(__name__)
 
@@ -13,3 +17,78 @@ log = logging.getLogger(__name__)
 def send_test_mail(email):
     user = datastore.find_user(email=email)
     mail.send(_("Test mail"), user, "test")
+
+
+@job("notify-inactive-users")
+def notify_inactive_users(self):
+    if not current_app.config["YEARS_OF_INACTIVITY_BEFORE_DEACTIVATION"]:
+        logging.warning(
+            "YEARS_OF_INACTIVITY_BEFORE_DEACTIVATION setting is not set, no deletion planned"
+        )
+        return
+    notification_comparison_date = (
+        datetime.utcnow()
+        - timedelta(days=current_app.config["YEARS_OF_INACTIVITY_BEFORE_DEACTIVATION"] * 365)
+        + timedelta(days=current_app.config["DAYS_BEFORE_ACCOUNT_INACTIVITY_NOTIFY_DELAY"])
+    )
+
+    users_to_notify = User.objects(
+        deleted=None,
+        inactive_deletion_notified_at=None,
+        current_login_at__lte=notification_comparison_date,
+    )
+    for i, user in enumerate(users_to_notify):
+        if i >= current_app.config["MAX_NUMBER_OF_USER_INACTIVITY_NOTIFICATIONS"]:
+            logging.warning("MAX_NUMBER_OF_USER_INACTIVITY_NOTIFICATIONS reached, stopping here.")
+            return
+        mail.send(
+            _("Inactivity of your {site} account").format(site=current_app.config["SITE_TITLE"]),
+            user,
+            "account_inactivity",
+            user=user,
+        )
+        logging.debug(f"Notified {user.email} of account inactivity")
+        user.inactive_deletion_notified_at = datetime.utcnow()
+        user.save()
+
+    logging.info(f"Notified {users_to_notify.count()} inactive users")
+
+
+@job("delete-inactive-users")
+def delete_inactive_users(self):
+    if not current_app.config["YEARS_OF_INACTIVITY_BEFORE_DEACTIVATION"]:
+        logging.warning(
+            "YEARS_OF_INACTIVITY_BEFORE_DEACTIVATION setting is not set, no deletion planned"
+        )
+        return
+
+    # Clear inactive_deletion_notified_at field if user has logged in since notification
+    for user in User.objects(deleted=None, inactive_deletion_notified_at__exists=True):
+        if user.current_login_at > user.inactive_deletion_notified_at:
+            user.inactive_deletion_notified_at = None
+            user.save()
+
+    # Delete inactive users upon notification delay if user still hasn't logged in
+    deletion_comparison_date = datetime.utcnow() - timedelta(
+        days=current_app.config["YEARS_OF_INACTIVITY_BEFORE_DEACTIVATION"] * 365
+    )
+    notified_at = datetime.utcnow() - timedelta(
+        days=current_app.config["DAYS_BEFORE_ACCOUNT_INACTIVITY_NOTIFY_DELAY"]
+    )
+    users_to_delete = User.objects(
+        deleted=None,
+        current_login_at__lte=deletion_comparison_date,
+        inactive_deletion_notified_at__lte=notified_at,
+    )
+    for user in users_to_delete:
+        copied_user = copy(user)
+        user.mark_as_deleted(notify=False, delete_comments=False)
+        logging.warning(f"Deleted user {copied_user.email} due to account inactivity")
+        mail.send(
+            _("Deletion of your inactive {site} account").format(
+                site=current_app.config["SITE_TITLE"]
+            ),
+            copied_user,
+            "inactive_account_deleted",
+        )
+    logging.info(f"Deleted {users_to_delete.count()} inactive users")

udata/core/user/tests/test_user_model.py

@@ -18,7 +18,7 @@ class UserModelTest:
         user = UserFactory()
         other_user = UserFactory()
         org = OrganizationFactory(editors=[user])
-        discussion_only_user = DiscussionFactory(
+        discussion = DiscussionFactory(
             user=user,
             subject=org,
             discussion=[
@@ -26,14 +26,6 @@ class UserModelTest:
                 MessageDiscussionFactory(posted_by=user),
             ],
         )
-        discussion_with_other = DiscussionFactory(
-            user=other_user,
-            subject=org,
-            discussion=[
-                MessageDiscussionFactory(posted_by=other_user),
-                MessageDiscussionFactory(posted_by=user),
-            ],
-        )
         user_follow_org = Follow.objects.create(follower=user, following=org)
         user_followed = Follow.objects.create(follower=other_user, following=user)
 
@@ -42,15 +34,40 @@ class UserModelTest:
         org.reload()
         assert len(org.members) == 0
 
-        assert Discussion.objects(id=discussion_only_user.id).first() is None
-        discussion_with_other.reload()
-        assert discussion_with_other.discussion[1].content == "DELETED"
+        # discussions are kept by default
+        discussion.reload()
+        assert len(discussion.discussion) == 2
+        assert discussion.discussion[1].content != "DELETED"
 
         assert Follow.objects(id=user_follow_org.id).first() is None
         assert Follow.objects(id=user_followed.id).first() is None
 
         assert user.slug == "deleted"
 
+    def test_mark_as_deleted_with_comments_deletion(self):
+        user = UserFactory()
+        other_user = UserFactory()
+        discussion_only_user = DiscussionFactory(
+            user=user,
+            discussion=[
+                MessageDiscussionFactory(posted_by=user),
+                MessageDiscussionFactory(posted_by=user),
+            ],
+        )
+        discussion_with_other = DiscussionFactory(
+            user=other_user,
+            discussion=[
+                MessageDiscussionFactory(posted_by=other_user),
+                MessageDiscussionFactory(posted_by=user),
+            ],
+        )
+
+        user.mark_as_deleted(delete_comments=True)
+
+        assert Discussion.objects(id=discussion_only_user.id).first() is None
+        discussion_with_other.reload()
+        assert discussion_with_other.discussion[1].content == "DELETED"
+
     def test_mark_as_deleted_slug_multiple(self):
         user = UserFactory()
         other_user = UserFactory()

udata/harvest/actions.py

@@ -7,6 +7,7 @@ from bson import ObjectId
 from flask import current_app
 
 from udata.auth import current_user
+from udata.core.dataservices.models import Dataservice
 from udata.core.dataset.models import HarvestDatasetMetadata
 from udata.models import Dataset, Organization, PeriodicTask, User
 from udata.storage.s3 import delete_file
@@ -18,6 +19,7 @@ from .models import (
     VALIDATION_REFUSED,
     HarvestJob,
     HarvestSource,
+    archive_harvested_dataservice,
     archive_harvested_dataset,
 )
 from .tasks import harvest
@@ -161,6 +163,9 @@ def purge_sources():
         datasets = Dataset.objects.filter(harvest__source_id=str(source.id))
         for dataset in datasets:
             archive_harvested_dataset(dataset, reason="harvester-deleted", dryrun=False)
+        dataservices = Dataservice.objects.filter(harvest__source_id=str(source.id))
+        for dataservice in dataservices:
+            archive_harvested_dataservice(dataservice, reason="harvester-deleted", dryrun=False)
         source.delete()
     return count
 

udata/harvest/backends/base.py

@@ -20,6 +20,7 @@ from ..models import (
     HarvestItem,
     HarvestJob,
     HarvestLog,
+    archive_harvested_dataservice,
     archive_harvested_dataset,
 )
 from ..signals import after_harvest_job, before_harvest_job
@@ -342,6 +343,7 @@ class BaseBackend(object):
         harvest.last_update = datetime.utcnow()
 
         harvest.archived_at = None
+        harvest.archived_reason = None
 
         return harvest
 
@@ -370,9 +372,10 @@ class BaseBackend(object):
             "harvest__remote_id__nin": remote_ids,
             "harvest__last_update__lt": limit_date,
         }
-        local_items_not_on_remote = Dataset.objects.filter(**q)
+        local_datasets_not_on_remote = Dataset.objects.filter(**q)
+        local_dataservices_not_on_remote = Dataservice.objects.filter(**q)
 
-        for dataset in local_items_not_on_remote:
+        for dataset in local_datasets_not_on_remote:
             if not dataset.harvest.archived_at:
                 archive_harvested_dataset(dataset, reason="not-on-remote", dryrun=self.dryrun)
             # add a HarvestItem to the job list (useful for report)
@@ -385,6 +388,23 @@ class BaseBackend(object):
 
             self.save_job()
 
+        for dataservice in local_dataservices_not_on_remote:
+            if not dataservice.harvest.archived_at:
+                archive_harvested_dataservice(
+                    dataservice, reason="not-on-remote", dryrun=self.dryrun
+                )
+            # add a HarvestItem to the job list (useful for report)
+            # even when archiving has already been done (useful for debug)
+            self.job.items.append(
+                HarvestItem(
+                    remote_id=str(dataservice.harvest.remote_id),
+                    dataservice=dataservice,
+                    status="archived",
+                )
+            )
+
+            self.save_job()
+
     def get_dataset(self, remote_id):
         """Get or create a dataset given its remote ID (and its source)
         We first try to match `source_id` to be source domain independent

udata/harvest/models.py

@@ -6,6 +6,7 @@ from urllib.parse import urlparse
 from werkzeug.utils import cached_property
 
 from udata.core.dataservices.models import Dataservice
+from udata.core.dataservices.models import HarvestMetadata as HarvestDataserviceMetadata
 from udata.core.dataset.models import HarvestDatasetMetadata
 from udata.core.owned import Owned, OwnedQuerySet
 from udata.i18n import lazy_gettext as _
@@ -203,3 +204,21 @@ def archive_harvested_dataset(dataset, reason, dryrun=False):
         dataset.validate()
     else:
         dataset.save()
+
+
+def archive_harvested_dataservice(dataservice, reason, dryrun=False):
+    """
+    Archive an harvested dataservice, setting extras accordingly.
+    If `dryrun` is True, the dataservice is not saved but validated only.
+    """
+    log.debug("Archiving dataservice %s", dataservice.id)
+    archival_date = datetime.utcnow()
+    dataservice.archived_at = archival_date
+    if not dataservice.harvest:
+        dataservice.harvest = HarvestDataserviceMetadata()
+    dataservice.harvest.archived_reason = reason
+    dataservice.harvest.archived_at = archival_date
+    if dryrun:
+        dataservice.validate()
+    else:
+        dataservice.save()

udata/harvest/tests/test_actions.py

@@ -6,6 +6,8 @@ from tempfile import NamedTemporaryFile
 import pytest
 from mock import patch
 
+from udata.core.dataservices.factories import DataserviceFactory
+from udata.core.dataservices.models import HarvestMetadata as HarvestDataserviceMetadata
 from udata.core.dataset.factories import DatasetFactory
 from udata.core.dataset.models import HarvestDatasetMetadata
 from udata.core.organization.factories import OrganizationFactory
@@ -396,17 +398,27 @@ class HarvestActionsTest:
         dataset_to_archive = DatasetFactory(
             harvest=HarvestDatasetMetadata(source_id=str(to_delete[0].id))
         )
+        dataservice_to_archive = DataserviceFactory(
+            harvest=HarvestDataserviceMetadata(source_id=str(to_delete[0].id))
+        )
 
         result = actions.purge_sources()
         dataset_to_archive.reload()
+        dataservice_to_archive.reload()
 
         assert result == len(to_delete)
         assert len(HarvestSource.objects) == len(to_keep)
         assert PeriodicTask.objects.filter(id=periodic_task.id).count() == 0
         assert HarvestJob.objects(id=harvest_job.id).count() == 0
+
         assert dataset_to_archive.harvest.archived == "harvester-deleted"
+        assert_equal_dates(dataset_to_archive.harvest.archived_at, now)
         assert_equal_dates(dataset_to_archive.archived, now)
 
+        assert dataservice_to_archive.harvest.archived_reason == "harvester-deleted"
+        assert_equal_dates(dataservice_to_archive.harvest.archived_at, now)
+        assert_equal_dates(dataservice_to_archive.archived_at, now)
+
     @pytest.mark.options(HARVEST_JOBS_RETENTION_DAYS=2)
     def test_purge_jobs(self):
         now = datetime.utcnow()

udata/harvest/tests/test_base_backend.py

@@ -4,6 +4,8 @@ from urllib.parse import urlparse
 import pytest
 from voluptuous import Schema
 
+from udata.core.dataservices.factories import DataserviceFactory
+from udata.core.dataservices.models import Dataservice
 from udata.core.dataset import tasks
 from udata.core.dataset.factories import DatasetFactory
 from udata.harvest.models import HarvestItem
@@ -20,9 +22,9 @@ class Unknown:
     pass
 
 
-def gen_remote_IDs(num: int) -> list[str]:
+def gen_remote_IDs(num: int, prefix: str = "") -> list[str]:
     """Generate remote IDs."""
-    return [f"fake-{i}" for i in range(num)]
+    return [f"{prefix}fake-{i}" for i in range(num)]
 
 
 class FakeBackend(BaseBackend):
@@ -45,6 +47,11 @@ class FakeBackend(BaseBackend):
             if self.is_done():
                 return
 
+        for remote_id in self.source.config.get("dataservice_remote_ids", []):
+            self.process_dataservice(remote_id)
+            if self.is_done():
+                return
+
     def inner_process_dataset(self, item: HarvestItem):
         dataset = self.get_dataset(item.remote_id)
 
@@ -55,6 +62,16 @@ class FakeBackend(BaseBackend):
             dataset.last_modified_internal = self.source.config["last_modified"]
         return dataset
 
+    def inner_process_dataservice(self, item: HarvestItem):
+        dataservice = self.get_dataservice(item.remote_id)
+
+        for key, value in DataserviceFactory.as_dict().items():
+            if getattr(dataservice, key) is None:
+                setattr(dataservice, key, value)
+        if self.source.config.get("last_modified"):
+            dataservice.last_modified_internal = self.source.config["last_modified"]
+        return dataservice
+
 
 class HarvestFilterTest:
     @pytest.mark.parametrize("type,expected", HarvestFilter.TYPES.items())
@@ -210,7 +227,13 @@ class BaseBackendTest:
 
     def test_autoarchive(self, app):
         nb_datasets = 3
-        source = HarvestSourceFactory(config={"dataset_remote_ids": gen_remote_IDs(nb_datasets)})
+        nb_dataservices = 3
+        source = HarvestSourceFactory(
+            config={
+                "dataset_remote_ids": gen_remote_IDs(nb_datasets, "dataset-"),
+                "dataservice_remote_ids": gen_remote_IDs(nb_dataservices, "dataservice-"),
+            }
+        )
         backend = FakeBackend(source)
 
         # create a dangling dataset to be archived
@@ -220,7 +243,15 @@ class BaseBackendTest:
             harvest={
                 "domain": source.domain,
                 "source_id": str(source.id),
-                "remote_id": "not-on-remote",
+                "remote_id": "dataset-not-on-remote",
+                "last_update": last_update,
+            }
+        )
+        dataservice_arch = DataserviceFactory(
+            harvest={
+                "domain": source.domain,
+                "source_id": str(source.id),
+                "remote_id": "dataservice-not-on-remote",
                 "last_update": last_update,
             }
         )
@@ -232,7 +263,15 @@ class BaseBackendTest:
             harvest={
                 "domain": source.domain,
                 "source_id": str(source.id),
-                "remote_id": "not-on-remote-two",
+                "remote_id": "dataset-not-on-remote-two",
+                "last_update": last_update,
+            }
+        )
+        dataservice_no_arch = DataserviceFactory(
+            harvest={
+                "domain": source.domain,
+                "source_id": str(source.id),
+                "remote_id": "dataservice-not-on-remote-two",
                 "last_update": last_update,
             }
         )
@@ -240,13 +279,17 @@ class BaseBackendTest:
         job = backend.harvest()
 
         # all datasets except arch : 3 mocks + 1 manual (no_arch)
-        assert len(job.items) == nb_datasets + 1
+        assert len(job.items) == (nb_datasets + 1) + (nb_dataservices + 1)
         # all datasets : 3 mocks + 2 manuals (arch and no_arch)
         assert Dataset.objects.count() == nb_datasets + 2
+        assert Dataservice.objects.count() == nb_dataservices + 2
 
         archived_items = [i for i in job.items if i.status == "archived"]
-        assert len(archived_items) == 1
+        assert len(archived_items) == 2
         assert archived_items[0].dataset == dataset_arch
+        assert archived_items[0].dataservice is None
+        assert archived_items[1].dataset is None
+        assert archived_items[1].dataservice == dataservice_arch
 
         dataset_arch.reload()
         assert dataset_arch.archived is not None
@@ -258,18 +301,41 @@ class BaseBackendTest:
         assert "archived" not in dataset_no_arch.harvest
         assert "archived_at" not in dataset_no_arch.harvest
 
+        dataservice_arch.reload()
+        assert dataservice_arch.archived_at is not None
+        assert "archived_reason" in dataservice_arch.harvest
+        assert "archived_at" in dataservice_arch.harvest
+
+        dataservice_no_arch.reload()
+        assert dataservice_no_arch.archived_at is None
+        assert "archived_reason" not in dataservice_no_arch.harvest
+        assert "archived_at" not in dataservice_no_arch.harvest
+
         # test unarchive: archive manually then relaunch harvest
-        dataset = Dataset.objects.get(**{"harvest__remote_id": "fake-1"})
+        dataset = Dataset.objects.get(**{"harvest__remote_id": "dataset-fake-1"})
         dataset.archived = datetime.utcnow()
         dataset.harvest.archived = "not-on-remote"
         dataset.harvest.archived_at = datetime.utcnow()
         dataset.save()
+
+        dataservice = Dataservice.objects.get(**{"harvest__remote_id": "dataservice-fake-1"})
+        dataservice.archived_at = datetime.utcnow()
+        dataservice.harvest.archived_reason = "not-on-remote"
+        dataservice.harvest.archived_at = datetime.utcnow()
+        dataservice.save()
+
         backend.harvest()
+
         dataset.reload()
         assert dataset.archived is None
         assert "archived" not in dataset.harvest
         assert "archived_at" not in dataset.harvest
 
+        dataservice.reload()
+        assert dataservice.archived_at is None
+        assert "archived_reason" not in dataservice.harvest
+        assert "archived_at" not in dataservice.harvest
+
     def test_harvest_datasets_get_deleted(self):
         nb_datasets = 3
         source = HarvestSourceFactory(config={"dataset_remote_ids": gen_remote_IDs(nb_datasets)})

udata/settings.py

@@ -115,6 +115,11 @@ class Defaults(object):
 
     SECURITY_RETURN_GENERIC_RESPONSES = False
 
+    # Inactive users settings
+    YEARS_OF_INACTIVITY_BEFORE_DELETION = None
+    DAYS_BEFORE_ACCOUNT_INACTIVITY_NOTIFY_DELAY = 30
+    MAX_NUMBER_OF_USER_INACTIVITY_NOTIFICATIONS = 200
+
     # Sentry configuration
     SENTRY_DSN = None
     SENTRY_TAGS = {}