udata 10.1.2.dev34172__py2.py3-none-any.whl → 10.1.3__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of udata might be problematic. Click here for more details.

Files changed (59) hide show
  1. udata/__init__.py +1 -1
  2. udata/commands/fixtures.py +1 -1
  3. udata/core/dataservices/constants.py +11 -0
  4. udata/core/dataservices/csv.py +3 -3
  5. udata/core/dataservices/models.py +27 -12
  6. udata/core/dataservices/rdf.py +5 -3
  7. udata/core/dataservices/search.py +13 -5
  8. udata/core/dataset/api.py +18 -3
  9. udata/core/dataset/forms.py +8 -4
  10. udata/core/dataset/models.py +6 -0
  11. udata/core/metrics/commands.py +20 -1
  12. udata/core/organization/api_fields.py +3 -1
  13. udata/core/user/api.py +8 -1
  14. udata/core/user/api_fields.py +5 -0
  15. udata/core/user/models.py +16 -11
  16. udata/core/user/tasks.py +81 -2
  17. udata/core/user/tests/test_user_model.py +29 -12
  18. udata/features/transfer/api.py +7 -4
  19. udata/harvest/actions.py +5 -0
  20. udata/harvest/backends/base.py +22 -2
  21. udata/harvest/models.py +19 -0
  22. udata/harvest/tests/test_actions.py +12 -0
  23. udata/harvest/tests/test_base_backend.py +74 -8
  24. udata/harvest/tests/test_dcat_backend.py +1 -1
  25. udata/migrations/2025-01-05-dataservices-fields-changes.py +136 -0
  26. udata/settings.py +5 -0
  27. udata/templates/mail/account_inactivity.html +29 -0
  28. udata/templates/mail/account_inactivity.txt +22 -0
  29. udata/templates/mail/inactive_account_deleted.html +5 -0
  30. udata/templates/mail/inactive_account_deleted.txt +6 -0
  31. udata/tests/api/test_dataservices_api.py +41 -2
  32. udata/tests/api/test_datasets_api.py +58 -0
  33. udata/tests/api/test_me_api.py +1 -1
  34. udata/tests/api/test_transfer_api.py +38 -0
  35. udata/tests/api/test_user_api.py +47 -8
  36. udata/tests/dataservice/test_csv_adapter.py +2 -0
  37. udata/tests/dataset/test_dataset_model.py +14 -0
  38. udata/tests/user/test_user_tasks.py +144 -0
  39. udata/translations/ar/LC_MESSAGES/udata.mo +0 -0
  40. udata/translations/ar/LC_MESSAGES/udata.po +88 -60
  41. udata/translations/de/LC_MESSAGES/udata.mo +0 -0
  42. udata/translations/de/LC_MESSAGES/udata.po +88 -60
  43. udata/translations/es/LC_MESSAGES/udata.mo +0 -0
  44. udata/translations/es/LC_MESSAGES/udata.po +88 -60
  45. udata/translations/fr/LC_MESSAGES/udata.mo +0 -0
  46. udata/translations/fr/LC_MESSAGES/udata.po +88 -60
  47. udata/translations/it/LC_MESSAGES/udata.mo +0 -0
  48. udata/translations/it/LC_MESSAGES/udata.po +88 -60
  49. udata/translations/pt/LC_MESSAGES/udata.mo +0 -0
  50. udata/translations/pt/LC_MESSAGES/udata.po +88 -60
  51. udata/translations/sr/LC_MESSAGES/udata.mo +0 -0
  52. udata/translations/sr/LC_MESSAGES/udata.po +88 -60
  53. udata/translations/udata.pot +83 -54
  54. {udata-10.1.2.dev34172.dist-info → udata-10.1.3.dist-info}/METADATA +15 -2
  55. {udata-10.1.2.dev34172.dist-info → udata-10.1.3.dist-info}/RECORD +59 -52
  56. {udata-10.1.2.dev34172.dist-info → udata-10.1.3.dist-info}/LICENSE +0 -0
  57. {udata-10.1.2.dev34172.dist-info → udata-10.1.3.dist-info}/WHEEL +0 -0
  58. {udata-10.1.2.dev34172.dist-info → udata-10.1.3.dist-info}/entry_points.txt +0 -0
  59. {udata-10.1.2.dev34172.dist-info → udata-10.1.3.dist-info}/top_level.txt +0 -0
udata/__init__.py CHANGED
@@ -4,5 +4,5 @@
4
4
  udata
5
5
  """
6
6
 
7
- __version__ = "10.1.2.dev"
7
+ __version__ = "10.1.3"
8
8
  __description__ = "Open data portal"
@@ -39,7 +39,7 @@ COMMUNITY_RES_URL = "/api/1/datasets/community_resources"
39
39
  DISCUSSION_URL = "/api/1/discussions"
40
40
 
41
41
 
42
- DEFAULT_FIXTURE_FILE_TAG: str = "v5.0.0"
42
+ DEFAULT_FIXTURE_FILE_TAG: str = "v6.0.0"
43
43
  DEFAULT_FIXTURE_FILE: str = f"https://raw.githubusercontent.com/opendatateam/udata-fixtures/{DEFAULT_FIXTURE_FILE_TAG}/results.json" # noqa
44
44
 
45
45
  DEFAULT_FIXTURES_RESULTS_FILENAME: str = "results.json"
@@ -0,0 +1,11 @@
1
+ DATASERVICE_FORMATS = ["REST", "WMS", "WSL"]
2
+
3
+
4
+ DATASERVICE_ACCESS_TYPE_OPEN = "open"
5
+ DATASERVICE_ACCESS_TYPE_OPEN_WITH_ACCOUNT = "open_with_account"
6
+ DATASERVICE_ACCESS_TYPE_RESTRICTED = "restricted"
7
+ DATASERVICE_ACCESS_TYPES = [
8
+ DATASERVICE_ACCESS_TYPE_OPEN,
9
+ DATASERVICE_ACCESS_TYPE_OPEN_WITH_ACCOUNT,
10
+ DATASERVICE_ACCESS_TYPE_RESTRICTED,
11
+ ]
@@ -13,13 +13,13 @@ class DataserviceCsvAdapter(csv.Adapter):
13
13
  ("url", lambda d: d.self_web_url()),
14
14
  "description",
15
15
  "base_api_url",
16
- "endpoint_description_url",
16
+ "machine_documentation_url",
17
+ "technical_documentation_url",
17
18
  "business_documentation_url",
18
19
  "authorization_request_url",
19
20
  "availability",
20
21
  "rate_limiting",
21
- "is_restricted",
22
- "has_token",
22
+ "access_type",
23
23
  "license",
24
24
  ("organization", "organization.name"),
25
25
  ("organization_id", "organization.id"),
@@ -8,6 +8,7 @@ from mongoengine.signals import post_save
8
8
  import udata.core.contact_point.api_fields as contact_api_fields
9
9
  import udata.core.dataset.api_fields as datasets_api_fields
10
10
  from udata.api_fields import field, function_field, generate_fields
11
+ from udata.core.dataservices.constants import DATASERVICE_ACCESS_TYPES, DATASERVICE_FORMATS
11
12
  from udata.core.dataset.models import Dataset
12
13
  from udata.core.metrics.models import WithMetrics
13
14
  from udata.core.owned import Owned, OwnedQuerySet
@@ -24,8 +25,6 @@ from udata.uris import endpoint_for
24
25
  # "spatial"
25
26
  # "temporal_coverage"
26
27
 
27
- DATASERVICE_FORMATS = ["REST", "WMS", "WSL"]
28
-
29
28
 
30
29
  class DataserviceQuerySet(OwnedQuerySet):
31
30
  def visible(self):
@@ -95,16 +94,23 @@ class HarvestMetadata(db.EmbeddedDocument):
95
94
  )
96
95
  last_update = field(db.DateTimeField(), description="Date of the last harvesting")
97
96
  archived_at = field(db.DateTimeField())
97
+ archived_reason = field(db.StringField())
98
98
 
99
99
 
100
100
  @generate_fields(
101
101
  searchable=True,
102
102
  additional_filters={"organization_badge": "organization.badges"},
103
+ additional_sorts=[
104
+ {"key": "followers", "value": "metrics.followers"},
105
+ {"key": "views", "value": "metrics.views"},
106
+ ],
103
107
  )
104
108
  class Dataservice(WithMetrics, Owned, db.Document):
105
109
  meta = {
106
110
  "indexes": [
107
111
  "$title",
112
+ "metrics.followers",
113
+ "metrics.views",
108
114
  ]
109
115
  + Owned.meta["indexes"],
110
116
  "queryset_class": DataserviceQuerySet,
@@ -132,13 +138,22 @@ class Dataservice(WithMetrics, Owned, db.Document):
132
138
  )
133
139
  description = field(db.StringField(default=""), description="In markdown")
134
140
  base_api_url = field(db.URLField(), sortable=True)
135
- endpoint_description_url = field(db.URLField())
141
+
142
+ machine_documentation_url = field(
143
+ db.URLField(), description="Swagger link, OpenAPI format, WMS XML…"
144
+ )
145
+ technical_documentation_url = field(db.URLField(), description="HTML version of a Swagger…")
136
146
  business_documentation_url = field(db.URLField())
137
- authorization_request_url = field(db.URLField())
138
- availability = field(db.FloatField(min=0, max=100), example="99.99")
147
+
139
148
  rate_limiting = field(db.StringField())
140
- is_restricted = field(db.BooleanField(), filterable={})
141
- has_token = field(db.BooleanField())
149
+ rate_limiting_url = field(db.URLField())
150
+
151
+ availability = field(db.FloatField(min=0, max=100), example="99.99")
152
+ availability_url = field(db.URLField())
153
+
154
+ access_type = field(db.StringField(choices=DATASERVICE_ACCESS_TYPES), filterable={})
155
+ authorization_request_url = field(db.URLField())
156
+
142
157
  format = field(db.StringField(choices=DATASERVICE_FORMATS))
143
158
 
144
159
  license = field(
@@ -223,11 +238,11 @@ class Dataservice(WithMetrics, Owned, db.Document):
223
238
  def self_web_url(self):
224
239
  return endpoint_for("dataservices.show", dataservice=self, _external=True)
225
240
 
226
- # TODO
227
- # frequency = db.StringField(choices=list(UPDATE_FREQUENCIES.keys()))
228
- # temporal_coverage = db.EmbeddedDocumentField(db.DateRange)
229
- # spatial = db.EmbeddedDocumentField(SpatialCoverage)
230
- # harvest = db.EmbeddedDocumentField(HarvestDatasetMetadata)
241
+ __metrics_keys__ = [
242
+ "discussions",
243
+ "followers",
244
+ "views",
245
+ ]
231
246
 
232
247
  @property
233
248
  def is_hidden(self):
@@ -42,7 +42,9 @@ def dataservice_from_rdf(
42
42
  dataservice.description = sanitize_html(d.value(DCT.description) or d.value(DCT.abstract))
43
43
 
44
44
  dataservice.base_api_url = url_from_rdf(d, DCAT.endpointURL)
45
- dataservice.endpoint_description_url = url_from_rdf(d, DCAT.endpointDescription)
45
+
46
+ # TODO detect if it's human-readable or not?
47
+ dataservice.machine_documentation_url = url_from_rdf(d, DCAT.endpointDescription)
46
48
 
47
49
  roles = [ # Imbricated list of contact points for each role
48
50
  contact_points_from_rdf(d, rdf_entity, role, dataservice)
@@ -145,8 +147,8 @@ def dataservice_to_rdf(dataservice: Dataservice, graph=None):
145
147
  ),
146
148
  )
147
149
 
148
- if dataservice.endpoint_description_url:
149
- d.set(DCAT.endpointDescription, URIRef(dataservice.endpoint_description_url))
150
+ if dataservice.machine_documentation_url:
151
+ d.set(DCAT.endpointDescription, URIRef(dataservice.machine_documentation_url))
150
152
 
151
153
  # Add DCAT-AP HVD properties if the dataservice is tagged hvd.
152
154
  # See https://semiceu.github.io/DCAT-AP/releases/2.2.0-hvd/
@@ -5,6 +5,11 @@ from flask_restx.inputs import boolean
5
5
 
6
6
  from udata.api import api
7
7
  from udata.api.parsers import ModelApiParser
8
+ from udata.core.dataservices.constants import (
9
+ DATASERVICE_ACCESS_TYPE_OPEN,
10
+ DATASERVICE_ACCESS_TYPE_OPEN_WITH_ACCOUNT,
11
+ DATASERVICE_ACCESS_TYPE_RESTRICTED,
12
+ )
8
13
  from udata.models import Dataservice, Organization, User
9
14
  from udata.search import (
10
15
  BoolFilter,
@@ -47,7 +52,11 @@ class DataserviceApiParser(ModelApiParser):
47
52
  api.abort(400, "Organization arg must be an identifier")
48
53
  dataservices = dataservices.filter(organization=args["organization"])
49
54
  if "is_restricted" in args:
50
- dataservices = dataservices.filter(is_restricted=boolean(args["is_restricted"]))
55
+ dataservices = dataservices.filter(
56
+ access_type__in=[DATASERVICE_ACCESS_TYPE_RESTRICTED]
57
+ if boolean(args["is_restricted"])
58
+ else [DATASERVICE_ACCESS_TYPE_OPEN, DATASERVICE_ACCESS_TYPE_OPEN_WITH_ACCOUNT]
59
+ )
51
60
  return dataservices
52
61
 
53
62
 
@@ -56,9 +65,7 @@ class DataserviceSearch(ModelSearchAdapter):
56
65
  model = Dataservice
57
66
  search_url = "dataservices/"
58
67
 
59
- sorts = {
60
- "created": "created_at",
61
- }
68
+ sorts = {"created": "created_at", "views": "views", "followers": "followers"}
62
69
 
63
70
  filters = {
64
71
  "tag": Filter(),
@@ -114,5 +121,6 @@ class DataserviceSearch(ModelSearchAdapter):
114
121
  "tags": dataservice.tags,
115
122
  "extras": extras,
116
123
  "followers": dataservice.metrics.get("followers", 0),
117
- "is_restricted": dataservice.is_restricted or False,
124
+ "is_restricted": dataservice.access_type == DATASERVICE_ACCESS_TYPE_RESTRICTED,
125
+ "views": dataservice.metrics.get("views", 0),
118
126
  }
udata/core/dataset/api.py CHANGED
@@ -62,7 +62,12 @@ from .exceptions import (
62
62
  SchemasCacheUnavailableException,
63
63
  SchemasCatalogNotFoundException,
64
64
  )
65
- from .forms import CommunityResourceForm, DatasetForm, ResourceForm, ResourcesListForm
65
+ from .forms import (
66
+ CommunityResourceForm,
67
+ DatasetForm,
68
+ ResourceFormWithoutId,
69
+ ResourcesListForm,
70
+ )
66
71
  from .models import (
67
72
  Checksum,
68
73
  CommunityResource,
@@ -379,8 +384,9 @@ class ResourcesAPI(API):
379
384
  def post(self, dataset):
380
385
  """Create a new resource for a given dataset"""
381
386
  ResourceEditPermission(dataset).test()
382
- form = api.validate(ResourceForm)
387
+ form = api.validate(ResourceFormWithoutId)
383
388
  resource = Resource()
389
+
384
390
  if form._fields.get("filetype").data != "remote":
385
391
  api.abort(400, "This endpoint only supports remote resources")
386
392
  form.populate_obj(resource)
@@ -545,10 +551,19 @@ class ResourceAPI(ResourceMixin, API):
545
551
  """Update a given resource on a given dataset"""
546
552
  ResourceEditPermission(dataset).test()
547
553
  resource = self.get_resource_or_404(dataset, rid)
548
- form = api.validate(ResourceForm, resource)
554
+ form = api.validate(ResourceFormWithoutId, resource)
555
+
556
+ # ensure filetype is not modified after creation
557
+ if (
558
+ form._fields.get("filetype").data
559
+ and form._fields.get("filetype").data != resource.filetype
560
+ ):
561
+ abort(400, "Cannot modify filetype after creation")
562
+
549
563
  # ensure API client does not override url on self-hosted resources
550
564
  if resource.filetype == "file":
551
565
  form._fields.get("url").data = resource.url
566
+
552
567
  # populate_obj populates existing resource object with the content of the form.
553
568
  # update_resource saves the updated resource dict to the database
554
569
  # the additional dataset.save is required as we update the last_modified date.
@@ -72,7 +72,7 @@ class BaseResourceForm(ModelForm):
72
72
  [validators.DataRequired()],
73
73
  choices=list(RESOURCE_FILETYPES.items()),
74
74
  default="file",
75
- description=_("Whether the resource is an uploaded file, " "a remote file or an API"),
75
+ description=_("Whether the resource is an uploaded file, a remote file or an API"),
76
76
  )
77
77
  type = fields.RadioField(
78
78
  _("Type"),
@@ -89,7 +89,7 @@ class BaseResourceForm(ModelForm):
89
89
  checksum = fields.FormField(ChecksumForm)
90
90
  mime = fields.StringField(
91
91
  _("Mime type"),
92
- description=_("The mime type associated to the extension. " "(ex: text/plain)"),
92
+ description=_("The mime type associated to the extension. (ex: text/plain)"),
93
93
  )
94
94
  filesize = fields.IntegerField(
95
95
  _("Size"), [validators.optional()], description=_("The file size in bytes")
@@ -104,6 +104,10 @@ class ResourceForm(BaseResourceForm):
104
104
  id = fields.UUIDField()
105
105
 
106
106
 
107
+ class ResourceFormWithoutId(BaseResourceForm):
108
+ model_class = Resource
109
+
110
+
107
111
  class CommunityResourceForm(BaseResourceForm):
108
112
  model_class = CommunityResource
109
113
 
@@ -145,7 +149,7 @@ class DatasetForm(ModelForm):
145
149
  description = fields.MarkdownField(
146
150
  _("Description"),
147
151
  [validators.DataRequired(), validators.Length(max=DESCRIPTION_SIZE_LIMIT)],
148
- description=_("The details about the dataset " "(collection process, specifics...)."),
152
+ description=_("The details about the dataset (collection process, specifics...)."),
149
153
  )
150
154
  license = fields.ModelSelectField(_("License"), model=License, allow_blank=True)
151
155
  frequency = fields.SelectField(
@@ -168,7 +172,7 @@ class DatasetForm(ModelForm):
168
172
  tags = fields.TagField(_("Tags"), description=_("Some taxonomy keywords"))
169
173
  private = fields.BooleanField(
170
174
  _("Private"),
171
- description=_("Restrict the dataset visibility to you or " "your organization only."),
175
+ description=_("Restrict the dataset visibility to you or your organization only."),
172
176
  )
173
177
 
174
178
  owner = fields.CurrentUserField()
@@ -638,6 +638,9 @@ class Dataset(WithMetrics, DatasetBadgeMixin, Owned, db.Document):
638
638
  if self.frequency in LEGACY_FREQUENCIES:
639
639
  self.frequency = LEGACY_FREQUENCIES[self.frequency]
640
640
 
641
+ if len(set(res.id for res in self.resources)) != len(self.resources):
642
+ raise MongoEngineValidationError(f"Duplicate resource ID in dataset #{self.id}.")
643
+
641
644
  for key, value in self.extras.items():
642
645
  if not key.startswith("custom:"):
643
646
  continue
@@ -897,6 +900,9 @@ class Dataset(WithMetrics, DatasetBadgeMixin, Owned, db.Document):
897
900
  def add_resource(self, resource):
898
901
  """Perform an atomic prepend for a new resource"""
899
902
  resource.validate()
903
+ if resource.id in [r.id for r in self.resources]:
904
+ raise MongoEngineValidationError("Cannot add resource with already existing ID")
905
+
900
906
  self.update(
901
907
  __raw__={"$push": {"resources": {"$each": [resource.to_mongo()], "$position": 0}}}
902
908
  )
@@ -4,6 +4,7 @@ import click
4
4
  from flask import current_app
5
5
 
6
6
  from udata.commands import cli, success
7
+ from udata.core.dataservices.models import Dataservice
7
8
  from udata.models import Dataset, GeoZone, Organization, Reuse, Site, User
8
9
 
9
10
  log = logging.getLogger(__name__)
@@ -19,6 +20,7 @@ def grp():
19
20
  @click.option("-s", "--site", is_flag=True, help="Update site metrics")
20
21
  @click.option("-o", "--organizations", is_flag=True, help="Compute organizations metrics")
21
22
  @click.option("-d", "--datasets", is_flag=True, help="Compute datasets metrics")
23
+ @click.option("--dataservices", is_flag=True, help="Compute dataservices metrics")
22
24
  @click.option("-r", "--reuses", is_flag=True, help="Compute reuses metrics")
23
25
  @click.option("-u", "--users", is_flag=True, help="Compute users metrics")
24
26
  @click.option("-g", "--geozones", is_flag=True, help="Compute geo levels metrics")
@@ -28,12 +30,13 @@ def update(
28
30
  organizations=False,
29
31
  users=False,
30
32
  datasets=False,
33
+ dataservices=False,
31
34
  reuses=False,
32
35
  geozones=False,
33
36
  drop=False,
34
37
  ):
35
38
  """Update all metrics for the current date"""
36
- do_all = not any((site, organizations, users, datasets, reuses, geozones))
39
+ do_all = not any((site, organizations, users, datasets, dataservices, reuses, geozones))
37
40
 
38
41
  if do_all or site:
39
42
  log.info("Update site metrics")
@@ -75,6 +78,22 @@ def update(
75
78
  log.info(f"Error during update: {e}")
76
79
  continue
77
80
 
81
+ if do_all or dataservices:
82
+ log.info("Update dataservices metrics")
83
+ all_dataservices = Dataservice.objects.visible().timeout(False)
84
+ with click.progressbar(
85
+ all_dataservices, length=Dataservice.objects.count()
86
+ ) as dataservice_bar:
87
+ for dataservice in dataservice_bar:
88
+ try:
89
+ if drop:
90
+ dataservice.metrics.clear()
91
+ dataservice.count_discussions()
92
+ dataservice.count_followers()
93
+ except Exception as e:
94
+ log.info(f"Error during update: {e}")
95
+ continue
96
+
78
97
  if do_all or reuses:
79
98
  log.info("Update reuses metrics")
80
99
  all_reuses = Reuse.objects.visible().timeout(False)
@@ -82,7 +82,9 @@ member_user_with_email_fields = api.inherit(
82
82
  readonly=True,
83
83
  ),
84
84
  "last_login_at": fields.Raw(
85
- attribute=lambda o: o.last_login_at if check_can_access_user_private_info() else None,
85
+ attribute=lambda o: o.current_login_at
86
+ if check_can_access_user_private_info()
87
+ else None,
86
88
  description="The user last connection date (only present on show organization endpoint if the current user is member of the organization: admin or editor)",
87
89
  readonly=True,
88
90
  ),
udata/core/user/api.py CHANGED
@@ -275,6 +275,13 @@ delete_parser.add_argument(
275
275
  location="args",
276
276
  default=False,
277
277
  )
278
+ delete_parser.add_argument(
279
+ "delete_comments",
280
+ type=bool,
281
+ help="Delete comments posted by the user upon user deletion",
282
+ location="args",
283
+ default=False,
284
+ )
278
285
 
279
286
 
280
287
  @ns.route("/<user:user>/", endpoint="user")
@@ -317,7 +324,7 @@ class UserAPI(API):
317
324
  403, "You cannot delete yourself with this API. " + 'Use the "me" API instead.'
318
325
  )
319
326
 
320
- user.mark_as_deleted(notify=not args["no_mail"])
327
+ user.mark_as_deleted(notify=not args["no_mail"], delete_comments=args["delete_comments"])
321
328
  return "", 204
322
329
 
323
330
 
@@ -61,6 +61,11 @@ user_fields = api.model(
61
61
  "since": fields.ISODateTime(
62
62
  attribute="created_at", description="The registeration date", required=True
63
63
  ),
64
+ "last_login_at": fields.Raw(
65
+ attribute=lambda o: o.current_login_at if current_user_is_admin_or_self() else None,
66
+ description="The user last connection date (only present for global admins and on /me)",
67
+ readonly=True,
68
+ ),
64
69
  "page": fields.UrlFor(
65
70
  "users.show",
66
71
  lambda u: {"user": u},
udata/core/user/models.py CHANGED
@@ -82,6 +82,10 @@ class User(WithMetrics, UserMixin, db.Document):
82
82
  ext = db.MapField(db.GenericEmbeddedDocumentField())
83
83
  extras = db.ExtrasField()
84
84
 
85
+ # Used to track notification for automatic inactive users deletion
86
+ # when YEARS_OF_INACTIVITY_BEFORE_DEACTIVATION is set
87
+ inactive_deletion_notified_at = db.DateTimeField()
88
+
85
89
  before_save = Signal()
86
90
  after_save = Signal()
87
91
  on_create = Signal()
@@ -237,7 +241,7 @@ class User(WithMetrics, UserMixin, db.Document):
237
241
  raise NotImplementedError("""This method should not be using directly.
238
242
  Use `mark_as_deleted` (or `_delete` if you know what you're doing)""")
239
243
 
240
- def mark_as_deleted(self, notify: bool = True):
244
+ def mark_as_deleted(self, notify: bool = True, delete_comments: bool = False):
241
245
  if self.avatar.filename is not None:
242
246
  storage = storages.avatars
243
247
  storage.delete(self.avatar.filename)
@@ -265,16 +269,17 @@ class User(WithMetrics, UserMixin, db.Document):
265
269
  member for member in organization.members if member.user != self
266
270
  ]
267
271
  organization.save()
268
- for discussion in Discussion.objects(discussion__posted_by=self):
269
- # Remove all discussions with current user as only participant
270
- if all(message.posted_by == self for message in discussion.discussion):
271
- discussion.delete()
272
- continue
273
-
274
- for message in discussion.discussion:
275
- if message.posted_by == self:
276
- message.content = "DELETED"
277
- discussion.save()
272
+ if delete_comments:
273
+ for discussion in Discussion.objects(discussion__posted_by=self):
274
+ # Remove all discussions with current user as only participant
275
+ if all(message.posted_by == self for message in discussion.discussion):
276
+ discussion.delete()
277
+ continue
278
+
279
+ for message in discussion.discussion:
280
+ if message.posted_by == self:
281
+ message.content = "DELETED"
282
+ discussion.save()
278
283
  Follow.objects(follower=self).delete()
279
284
  Follow.objects(following=self).delete()
280
285
 
udata/core/user/tasks.py CHANGED
@@ -1,10 +1,14 @@
1
1
  import logging
2
+ from copy import copy
3
+ from datetime import datetime, timedelta
4
+
5
+ from flask import current_app
2
6
 
3
7
  from udata import mail
4
8
  from udata.i18n import lazy_gettext as _
5
- from udata.tasks import task
9
+ from udata.tasks import job, task
6
10
 
7
- from .models import datastore
11
+ from .models import User, datastore
8
12
 
9
13
  log = logging.getLogger(__name__)
10
14
 
@@ -13,3 +17,78 @@ log = logging.getLogger(__name__)
13
17
  def send_test_mail(email):
14
18
  user = datastore.find_user(email=email)
15
19
  mail.send(_("Test mail"), user, "test")
20
+
21
+
22
+ @job("notify-inactive-users")
23
+ def notify_inactive_users(self):
24
+ if not current_app.config["YEARS_OF_INACTIVITY_BEFORE_DEACTIVATION"]:
25
+ logging.warning(
26
+ "YEARS_OF_INACTIVITY_BEFORE_DEACTIVATION setting is not set, no deletion planned"
27
+ )
28
+ return
29
+ notification_comparison_date = (
30
+ datetime.utcnow()
31
+ - timedelta(days=current_app.config["YEARS_OF_INACTIVITY_BEFORE_DEACTIVATION"] * 365)
32
+ + timedelta(days=current_app.config["DAYS_BEFORE_ACCOUNT_INACTIVITY_NOTIFY_DELAY"])
33
+ )
34
+
35
+ users_to_notify = User.objects(
36
+ deleted=None,
37
+ inactive_deletion_notified_at=None,
38
+ current_login_at__lte=notification_comparison_date,
39
+ )
40
+ for i, user in enumerate(users_to_notify):
41
+ if i >= current_app.config["MAX_NUMBER_OF_USER_INACTIVITY_NOTIFICATIONS"]:
42
+ logging.warning("MAX_NUMBER_OF_USER_INACTIVITY_NOTIFICATIONS reached, stopping here.")
43
+ return
44
+ mail.send(
45
+ _("Inactivity of your {site} account").format(site=current_app.config["SITE_TITLE"]),
46
+ user,
47
+ "account_inactivity",
48
+ user=user,
49
+ )
50
+ logging.debug(f"Notified {user.email} of account inactivity")
51
+ user.inactive_deletion_notified_at = datetime.utcnow()
52
+ user.save()
53
+
54
+ logging.info(f"Notified {users_to_notify.count()} inactive users")
55
+
56
+
57
+ @job("delete-inactive-users")
58
+ def delete_inactive_users(self):
59
+ if not current_app.config["YEARS_OF_INACTIVITY_BEFORE_DEACTIVATION"]:
60
+ logging.warning(
61
+ "YEARS_OF_INACTIVITY_BEFORE_DEACTIVATION setting is not set, no deletion planned"
62
+ )
63
+ return
64
+
65
+ # Clear inactive_deletion_notified_at field if user has logged in since notification
66
+ for user in User.objects(deleted=None, inactive_deletion_notified_at__exists=True):
67
+ if user.current_login_at > user.inactive_deletion_notified_at:
68
+ user.inactive_deletion_notified_at = None
69
+ user.save()
70
+
71
+ # Delete inactive users upon notification delay if user still hasn't logged in
72
+ deletion_comparison_date = datetime.utcnow() - timedelta(
73
+ days=current_app.config["YEARS_OF_INACTIVITY_BEFORE_DEACTIVATION"] * 365
74
+ )
75
+ notified_at = datetime.utcnow() - timedelta(
76
+ days=current_app.config["DAYS_BEFORE_ACCOUNT_INACTIVITY_NOTIFY_DELAY"]
77
+ )
78
+ users_to_delete = User.objects(
79
+ deleted=None,
80
+ current_login_at__lte=deletion_comparison_date,
81
+ inactive_deletion_notified_at__lte=notified_at,
82
+ )
83
+ for user in users_to_delete:
84
+ copied_user = copy(user)
85
+ user.mark_as_deleted(notify=False, delete_comments=False)
86
+ logging.warning(f"Deleted user {copied_user.email} due to account inactivity")
87
+ mail.send(
88
+ _("Deletion of your inactive {site} account").format(
89
+ site=current_app.config["SITE_TITLE"]
90
+ ),
91
+ copied_user,
92
+ "inactive_account_deleted",
93
+ )
94
+ logging.info(f"Deleted {users_to_delete.count()} inactive users")
@@ -18,7 +18,7 @@ class UserModelTest:
18
18
  user = UserFactory()
19
19
  other_user = UserFactory()
20
20
  org = OrganizationFactory(editors=[user])
21
- discussion_only_user = DiscussionFactory(
21
+ discussion = DiscussionFactory(
22
22
  user=user,
23
23
  subject=org,
24
24
  discussion=[
@@ -26,14 +26,6 @@ class UserModelTest:
26
26
  MessageDiscussionFactory(posted_by=user),
27
27
  ],
28
28
  )
29
- discussion_with_other = DiscussionFactory(
30
- user=other_user,
31
- subject=org,
32
- discussion=[
33
- MessageDiscussionFactory(posted_by=other_user),
34
- MessageDiscussionFactory(posted_by=user),
35
- ],
36
- )
37
29
  user_follow_org = Follow.objects.create(follower=user, following=org)
38
30
  user_followed = Follow.objects.create(follower=other_user, following=user)
39
31
 
@@ -42,15 +34,40 @@ class UserModelTest:
42
34
  org.reload()
43
35
  assert len(org.members) == 0
44
36
 
45
- assert Discussion.objects(id=discussion_only_user.id).first() is None
46
- discussion_with_other.reload()
47
- assert discussion_with_other.discussion[1].content == "DELETED"
37
+ # discussions are kept by default
38
+ discussion.reload()
39
+ assert len(discussion.discussion) == 2
40
+ assert discussion.discussion[1].content != "DELETED"
48
41
 
49
42
  assert Follow.objects(id=user_follow_org.id).first() is None
50
43
  assert Follow.objects(id=user_followed.id).first() is None
51
44
 
52
45
  assert user.slug == "deleted"
53
46
 
47
+ def test_mark_as_deleted_with_comments_deletion(self):
48
+ user = UserFactory()
49
+ other_user = UserFactory()
50
+ discussion_only_user = DiscussionFactory(
51
+ user=user,
52
+ discussion=[
53
+ MessageDiscussionFactory(posted_by=user),
54
+ MessageDiscussionFactory(posted_by=user),
55
+ ],
56
+ )
57
+ discussion_with_other = DiscussionFactory(
58
+ user=other_user,
59
+ discussion=[
60
+ MessageDiscussionFactory(posted_by=other_user),
61
+ MessageDiscussionFactory(posted_by=user),
62
+ ],
63
+ )
64
+
65
+ user.mark_as_deleted(delete_comments=True)
66
+
67
+ assert Discussion.objects(id=discussion_only_user.id).first() is None
68
+ discussion_with_other.reload()
69
+ assert discussion_with_other.discussion[1].content == "DELETED"
70
+
54
71
  def test_mark_as_deleted_slug_multiple(self):
55
72
  user = UserFactory()
56
73
  other_user = UserFactory()
@@ -1,4 +1,4 @@
1
- from flask import request
1
+ from flask import abort, request
2
2
 
3
3
  from udata.api import API, api, base_reference, fields
4
4
  from udata.core.dataservices.models import Dataservice
@@ -27,7 +27,7 @@ transfer_request_fields = api.model(
27
27
  "recipient": fields.Nested(
28
28
  base_reference,
29
29
  required=True,
30
- description=("The transfer recipient, " "either an user or an organization"),
30
+ description=("The transfer recipient, either an user or an organization"),
31
31
  ),
32
32
  "comment": fields.String(
33
33
  description="An explanation about the transfer request", required=True
@@ -68,12 +68,12 @@ transfer_fields = api.model(
68
68
  "owner": fields.Polymorph(
69
69
  person_mapping,
70
70
  readonly=True,
71
- description=("The user or organization currently owning " "the transfered object"),
71
+ description=("The user or organization currently owning the transfered object"),
72
72
  ),
73
73
  "recipient": fields.Polymorph(
74
74
  person_mapping,
75
75
  readonly=True,
76
- description=("The user or organization receiving " "the transfered object"),
76
+ description=("The user or organization receiving the transfered object"),
77
77
  ),
78
78
  "subject": fields.Polymorph(
79
79
  subject_mapping, readonly=True, description="The transfered object"
@@ -186,6 +186,9 @@ class TransferRequestAPI(API):
186
186
  """Respond to a transfer request"""
187
187
  transfer = Transfer.objects.get_or_404(id=id_or_404(id))
188
188
 
189
+ if transfer.status != "pending":
190
+ abort(400, "Cannot update transfer after accepting/refusing")
191
+
189
192
  data = request.json
190
193
  comment = data.get("comment")
191
194