udata 14.0.0__py3-none-any.whl → 14.5.1.dev6__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of udata might be problematic. Click here for more details.

Files changed (152) hide show
  1. udata/api/__init__.py +2 -0
  2. udata/api_fields.py +35 -4
  3. udata/app.py +18 -20
  4. udata/auth/__init__.py +29 -6
  5. udata/auth/forms.py +2 -2
  6. udata/auth/views.py +13 -6
  7. udata/commands/dcat.py +1 -1
  8. udata/commands/serve.py +3 -11
  9. udata/commands/tests/test_fixtures.py +9 -9
  10. udata/core/access_type/api.py +1 -1
  11. udata/core/access_type/constants.py +12 -8
  12. udata/core/activity/api.py +5 -6
  13. udata/core/badges/tests/test_commands.py +6 -6
  14. udata/core/csv.py +5 -0
  15. udata/core/dataservices/api.py +8 -1
  16. udata/core/dataservices/apiv2.py +2 -5
  17. udata/core/dataservices/models.py +5 -2
  18. udata/core/dataservices/rdf.py +2 -1
  19. udata/core/dataservices/tasks.py +13 -2
  20. udata/core/dataset/api.py +10 -0
  21. udata/core/dataset/models.py +6 -6
  22. udata/core/dataset/permissions.py +31 -0
  23. udata/core/dataset/rdf.py +8 -2
  24. udata/core/dataset/tasks.py +23 -7
  25. udata/core/discussions/api.py +15 -1
  26. udata/core/discussions/models.py +6 -0
  27. udata/core/legal/__init__.py +0 -0
  28. udata/core/legal/mails.py +128 -0
  29. udata/core/organization/api.py +16 -5
  30. udata/core/organization/apiv2.py +2 -3
  31. udata/core/organization/mails.py +1 -1
  32. udata/core/organization/models.py +15 -2
  33. udata/core/organization/notifications.py +84 -0
  34. udata/core/organization/permissions.py +1 -1
  35. udata/core/organization/tasks.py +3 -0
  36. udata/core/pages/tests/test_api.py +32 -0
  37. udata/core/post/api.py +24 -69
  38. udata/core/post/models.py +84 -16
  39. udata/core/post/tests/test_api.py +24 -1
  40. udata/core/reports/api.py +18 -0
  41. udata/core/reports/models.py +42 -2
  42. udata/core/reuse/api.py +8 -0
  43. udata/core/reuse/apiv2.py +2 -5
  44. udata/core/reuse/models.py +1 -1
  45. udata/core/reuse/tasks.py +7 -0
  46. udata/core/spatial/forms.py +2 -2
  47. udata/core/topic/models.py +8 -2
  48. udata/core/user/api.py +10 -3
  49. udata/core/user/models.py +12 -2
  50. udata/features/notifications/api.py +7 -18
  51. udata/features/notifications/models.py +56 -0
  52. udata/features/notifications/tasks.py +25 -0
  53. udata/flask_mongoengine/engine.py +0 -4
  54. udata/flask_mongoengine/pagination.py +1 -1
  55. udata/frontend/markdown.py +2 -1
  56. udata/harvest/actions.py +21 -1
  57. udata/harvest/api.py +25 -8
  58. udata/harvest/backends/base.py +27 -1
  59. udata/harvest/backends/ckan/harvesters.py +11 -2
  60. udata/harvest/backends/dcat.py +4 -1
  61. udata/harvest/commands.py +33 -0
  62. udata/harvest/filters.py +17 -6
  63. udata/harvest/models.py +16 -0
  64. udata/harvest/permissions.py +27 -0
  65. udata/harvest/tests/ckan/test_ckan_backend.py +33 -0
  66. udata/harvest/tests/test_actions.py +58 -5
  67. udata/harvest/tests/test_api.py +276 -122
  68. udata/harvest/tests/test_base_backend.py +86 -1
  69. udata/harvest/tests/test_dcat_backend.py +81 -10
  70. udata/harvest/tests/test_filters.py +6 -0
  71. udata/i18n.py +1 -4
  72. udata/mail.py +19 -1
  73. udata/migrations/2025-10-31-create-membership-request-notifications.py +55 -0
  74. udata/migrations/2025-12-04-add-uuid-to-discussion-messages.py +28 -0
  75. udata/mongo/slug_fields.py +1 -1
  76. udata/rdf.py +58 -10
  77. udata/routing.py +2 -2
  78. udata/settings.py +11 -0
  79. udata/tasks.py +1 -0
  80. udata/templates/mail/message.html +5 -31
  81. udata/tests/__init__.py +27 -2
  82. udata/tests/api/__init__.py +108 -21
  83. udata/tests/api/test_activities_api.py +36 -0
  84. udata/tests/api/test_auth_api.py +121 -95
  85. udata/tests/api/test_base_api.py +7 -4
  86. udata/tests/api/test_datasets_api.py +50 -19
  87. udata/tests/api/test_organizations_api.py +192 -197
  88. udata/tests/api/test_reports_api.py +157 -0
  89. udata/tests/api/test_reuses_api.py +147 -147
  90. udata/tests/api/test_security_api.py +12 -12
  91. udata/tests/api/test_swagger.py +4 -4
  92. udata/tests/api/test_tags_api.py +8 -8
  93. udata/tests/api/test_user_api.py +1 -1
  94. udata/tests/apiv2/test_search.py +30 -0
  95. udata/tests/apiv2/test_swagger.py +4 -4
  96. udata/tests/cli/test_cli_base.py +8 -9
  97. udata/tests/dataservice/test_dataservice_tasks.py +29 -0
  98. udata/tests/dataset/test_dataset_commands.py +4 -4
  99. udata/tests/dataset/test_dataset_model.py +66 -26
  100. udata/tests/dataset/test_dataset_rdf.py +99 -5
  101. udata/tests/dataset/test_dataset_tasks.py +25 -0
  102. udata/tests/frontend/test_auth.py +58 -1
  103. udata/tests/frontend/test_csv.py +0 -3
  104. udata/tests/helpers.py +31 -27
  105. udata/tests/organization/test_notifications.py +67 -2
  106. udata/tests/plugin.py +6 -261
  107. udata/tests/search/test_search_integration.py +33 -0
  108. udata/tests/site/test_site_csv_exports.py +22 -10
  109. udata/tests/test_activity.py +9 -9
  110. udata/tests/test_api_fields.py +10 -0
  111. udata/tests/test_dcat_commands.py +2 -2
  112. udata/tests/test_discussions.py +5 -5
  113. udata/tests/test_legal_mails.py +359 -0
  114. udata/tests/test_migrations.py +21 -21
  115. udata/tests/test_notifications.py +15 -57
  116. udata/tests/test_notifications_task.py +43 -0
  117. udata/tests/test_owned.py +81 -1
  118. udata/tests/test_storages.py +25 -19
  119. udata/tests/test_topics.py +77 -61
  120. udata/tests/test_uris.py +33 -0
  121. udata/tests/workers/test_jobs_commands.py +23 -23
  122. udata/translations/ar/LC_MESSAGES/udata.mo +0 -0
  123. udata/translations/ar/LC_MESSAGES/udata.po +187 -108
  124. udata/translations/de/LC_MESSAGES/udata.mo +0 -0
  125. udata/translations/de/LC_MESSAGES/udata.po +187 -108
  126. udata/translations/es/LC_MESSAGES/udata.mo +0 -0
  127. udata/translations/es/LC_MESSAGES/udata.po +187 -108
  128. udata/translations/fr/LC_MESSAGES/udata.mo +0 -0
  129. udata/translations/fr/LC_MESSAGES/udata.po +188 -109
  130. udata/translations/it/LC_MESSAGES/udata.mo +0 -0
  131. udata/translations/it/LC_MESSAGES/udata.po +187 -108
  132. udata/translations/pt/LC_MESSAGES/udata.mo +0 -0
  133. udata/translations/pt/LC_MESSAGES/udata.po +187 -108
  134. udata/translations/sr/LC_MESSAGES/udata.mo +0 -0
  135. udata/translations/sr/LC_MESSAGES/udata.po +187 -108
  136. udata/translations/udata.pot +215 -106
  137. udata/uris.py +0 -2
  138. udata-14.5.1.dev6.dist-info/METADATA +109 -0
  139. {udata-14.0.0.dist-info → udata-14.5.1.dev6.dist-info}/RECORD +143 -140
  140. udata/core/post/forms.py +0 -30
  141. udata/flask_mongoengine/json.py +0 -38
  142. udata/templates/mail/base.html +0 -105
  143. udata/templates/mail/base.txt +0 -6
  144. udata/templates/mail/button.html +0 -3
  145. udata/templates/mail/layouts/1-column.html +0 -19
  146. udata/templates/mail/layouts/2-columns.html +0 -20
  147. udata/templates/mail/layouts/center-panel.html +0 -16
  148. udata-14.0.0.dist-info/METADATA +0 -132
  149. {udata-14.0.0.dist-info → udata-14.5.1.dev6.dist-info}/WHEEL +0 -0
  150. {udata-14.0.0.dist-info → udata-14.5.1.dev6.dist-info}/entry_points.txt +0 -0
  151. {udata-14.0.0.dist-info → udata-14.5.1.dev6.dist-info}/licenses/LICENSE +0 -0
  152. {udata-14.0.0.dist-info → udata-14.5.1.dev6.dist-info}/top_level.txt +0 -0
@@ -31,6 +31,7 @@ def dataservice_from_rdf(
31
31
  node,
32
32
  all_datasets: list[Dataset],
33
33
  remote_url_prefix: str | None = None,
34
+ dryrun: bool = False,
34
35
  ) -> Dataservice:
35
36
  """
36
37
  Create or update a dataservice from a RDF/DCAT graph
@@ -51,7 +52,7 @@ def dataservice_from_rdf(
51
52
  dataservice.machine_documentation_url = url_from_rdf(d, DCAT.endpointDescription)
52
53
 
53
54
  roles = [ # Imbricated list of contact points for each role
54
- contact_points_from_rdf(d, rdf_entity, role, dataservice)
55
+ contact_points_from_rdf(d, rdf_entity, role, dataservice, dryrun=dryrun)
55
56
  for rdf_entity, role in CONTACT_POINT_ENTITY_TO_ROLE.items()
56
57
  ]
57
58
  dataservice.contact_points = [ # Flattened list of contact points
@@ -6,6 +6,7 @@ from udata.core.constants import HVD
6
6
  from udata.core.dataservices.models import Dataservice
7
7
  from udata.core.organization.constants import CERTIFIED, PUBLIC_SERVICE
8
8
  from udata.core.organization.models import Organization
9
+ from udata.core.pages.models import Page
9
10
  from udata.core.topic.models import TopicElement
10
11
  from udata.harvest.models import HarvestJob
11
12
  from udata.models import Discussion, Follow, Transfer
@@ -22,12 +23,22 @@ def purge_dataservices(self):
22
23
  Follow.objects(following=dataservice).delete()
23
24
  # Remove discussions
24
25
  Discussion.objects(subject=dataservice).delete()
25
- # Remove HarvestItem references
26
- HarvestJob.objects(items__dataservice=dataservice).update(set__items__S__dataservice=None)
26
+ # Remove HarvestItem references (using update_many with array_filters to update all matching items)
27
+ HarvestJob._get_collection().update_many(
28
+ {"items.dataservice": dataservice.id},
29
+ {"$set": {"items.$[item].dataservice": None}},
30
+ array_filters=[{"item.dataservice": dataservice.id}],
31
+ )
27
32
  # Remove associated Transfers
28
33
  Transfer.objects(subject=dataservice).delete()
29
34
  # Remove dataservices references in Topics
30
35
  TopicElement.objects(element=dataservice).update(element=None)
36
+ # Remove dataservices in pages (mongoengine doesn't support updating a field in a generic embed)
37
+ Page._get_collection().update_many(
38
+ {"blocs.dataservices": dataservice.id},
39
+ {"$pull": {"blocs.$[b].dataservices": dataservice.id}},
40
+ array_filters=[{"b.dataservices": dataservice.id}],
41
+ )
31
42
  # Remove dataservice
32
43
  dataservice.delete()
33
44
 
udata/core/dataset/api.py CHANGED
@@ -39,6 +39,7 @@ from udata.core.dataservices.models import Dataservice
39
39
  from udata.core.dataset.models import CHECKSUM_TYPES
40
40
  from udata.core.followers.api import FollowAPI
41
41
  from udata.core.followers.models import Follow
42
+ from udata.core.legal.mails import add_send_legal_notice_argument, send_legal_notice_on_deletion
42
43
  from udata.core.organization.models import Organization
43
44
  from udata.core.reuse.models import Reuse
44
45
  from udata.core.storages.api import handle_upload, upload_parser
@@ -364,6 +365,9 @@ class DatasetsAtomFeedAPI(API):
364
365
  return response
365
366
 
366
367
 
368
+ dataset_delete_parser = add_send_legal_notice_argument(api.parser())
369
+
370
+
367
371
  @ns.route("/<dataset:dataset>/", endpoint="dataset", doc=common_doc)
368
372
  @api.response(404, "Dataset not found")
369
373
  @api.response(410, "Dataset has been deleted")
@@ -397,12 +401,16 @@ class DatasetAPI(API):
397
401
 
398
402
  @api.secure
399
403
  @api.doc("delete_dataset")
404
+ @api.expect(dataset_delete_parser)
400
405
  @api.response(204, "Dataset deleted")
401
406
  def delete(self, dataset):
402
407
  """Delete a dataset given its identifier"""
408
+ args = dataset_delete_parser.parse_args()
403
409
  if dataset.deleted:
404
410
  api.abort(410, "Dataset has been deleted")
405
411
  dataset.permissions["delete"].test()
412
+ send_legal_notice_on_deletion(dataset, args)
413
+
406
414
  dataset.deleted = datetime.utcnow()
407
415
  dataset.last_modified_internal = datetime.utcnow()
408
416
  dataset.save()
@@ -531,6 +539,8 @@ class ResourcesAPI(API):
531
539
  f"All resources must be reordered, you provided {len(resources)} "
532
540
  f"out of {len(dataset.resources)}",
533
541
  )
542
+ if any(isinstance(r, dict) and "id" not in r for r in resources):
543
+ api.abort(400, "Each resource must have an 'id' field")
534
544
  if set(r["id"] if isinstance(r, dict) else r for r in resources) != set(
535
545
  str(r.id) for r in dataset.resources
536
546
  ):
@@ -546,7 +546,10 @@ class Dataset(
546
546
  ),
547
547
  auditable=False,
548
548
  )
549
- description = field(db.StringField(required=True, default=""))
549
+ description = field(
550
+ db.StringField(required=True, default=""),
551
+ markdown=True,
552
+ )
550
553
  description_short = field(db.StringField(max_length=DESCRIPTION_SHORT_SIZE_LIMIT))
551
554
  license = field(db.ReferenceField("License"))
552
555
 
@@ -730,7 +733,7 @@ class Dataset(
730
733
  }
731
734
 
732
735
  def self_web_url(self, **kwargs):
733
- return cdata_url(f"/datasets/{self._link_id(**kwargs)}/", **kwargs)
736
+ return cdata_url(f"/datasets/{self._link_id(**kwargs)}", **kwargs)
734
737
 
735
738
  def self_api_url(self, **kwargs):
736
739
  return url_for(
@@ -795,7 +798,7 @@ class Dataset(
795
798
  Resources should be fetched when calling this method.
796
799
  """
797
800
  if self.harvest and self.harvest.modified_at:
798
- return self.harvest.modified_at
801
+ return to_naive_datetime(self.harvest.modified_at)
799
802
  if self.resources:
800
803
  return max([res.last_modified for res in self.resources])
801
804
  else:
@@ -1148,9 +1151,6 @@ class ResourceSchema(object):
1148
1151
  except requests.exceptions.RequestException as err:
1149
1152
  log.exception(f"Error while getting schema catalog from {endpoint}: {err}")
1150
1153
  schemas = cache.get(cache_key)
1151
- except requests.exceptions.JSONDecodeError as err:
1152
- log.exception(f"Error while getting schema catalog from {endpoint}: {err}")
1153
- schemas = cache.get(cache_key)
1154
1154
  else:
1155
1155
  schemas = data.get("schemas", [])
1156
1156
  cache.set(cache_key, schemas)
@@ -1,3 +1,6 @@
1
+ from flask_principal import Permission as BasePermission
2
+ from flask_principal import RoleNeed
3
+
1
4
  from udata.auth import Permission, UserNeed
2
5
  from udata.core.organization.permissions import (
3
6
  OrganizationAdminNeed,
@@ -22,6 +25,34 @@ class OwnablePermission(Permission):
22
25
  super(OwnablePermission, self).__init__(*needs)
23
26
 
24
27
 
28
+ class OwnableReadPermission(BasePermission):
29
+ """Permission to read a private ownable object.
30
+
31
+ Always grants access if the object is not private.
32
+ For private objects, requires owner, org member, or sysadmin.
33
+
34
+ We inherit from BasePermission instead of udata's Permission because
35
+ Permission automatically adds RoleNeed("admin") to all needs. This means
36
+ a permission with no needs would only allow admins. With BasePermission,
37
+ an empty needs set allows everyone (Flask-Principal returns True when
38
+ self.needs is empty).
39
+ """
40
+
41
+ def __init__(self, obj):
42
+ if not getattr(obj, "private", False):
43
+ super().__init__()
44
+ return
45
+
46
+ needs = [RoleNeed("admin")]
47
+ if obj.organization:
48
+ needs.append(OrganizationAdminNeed(obj.organization.id))
49
+ needs.append(OrganizationEditorNeed(obj.organization.id))
50
+ elif obj.owner:
51
+ needs.append(UserNeed(obj.owner.fs_uniquifier))
52
+
53
+ super().__init__(*needs)
54
+
55
+
25
56
  class DatasetEditPermission(OwnablePermission):
26
57
  """Permissions to edit a Dataset"""
27
58
 
udata/core/dataset/rdf.py CHANGED
@@ -742,7 +742,13 @@ def resource_from_rdf(graph_or_distrib, dataset=None, is_additionnal=False):
742
742
  return resource
743
743
 
744
744
 
745
- def dataset_from_rdf(graph: Graph, dataset=None, node=None, remote_url_prefix: str | None = None):
745
+ def dataset_from_rdf(
746
+ graph: Graph,
747
+ dataset=None,
748
+ node=None,
749
+ remote_url_prefix: str | None = None,
750
+ dryrun: bool = False,
751
+ ):
746
752
  """
747
753
  Create or update a dataset from a RDF/DCAT graph
748
754
  """
@@ -764,7 +770,7 @@ def dataset_from_rdf(graph: Graph, dataset=None, node=None, remote_url_prefix: s
764
770
  dataset.description = sanitize_html(description)
765
771
  dataset.frequency = frequency_from_rdf(d.value(DCT.accrualPeriodicity)) or dataset.frequency
766
772
  roles = [ # Imbricated list of contact points for each role
767
- contact_points_from_rdf(d, rdf_entity, role, dataset)
773
+ contact_points_from_rdf(d, rdf_entity, role, dataset, dryrun=dryrun)
768
774
  for rdf_entity, role in CONTACT_POINT_ENTITY_TO_ROLE.items()
769
775
  ]
770
776
  dataset.contact_points = [ # Flattened list of contact points
@@ -1,4 +1,5 @@
1
1
  import collections
2
+ import gzip
2
3
  import os
3
4
  from datetime import date, datetime
4
5
  from tempfile import NamedTemporaryFile
@@ -15,6 +16,7 @@ from udata.core.dataservices.models import Dataservice
15
16
  from udata.core.dataset.constants import INSPIRE
16
17
  from udata.core.organization.constants import CERTIFIED, PUBLIC_SERVICE
17
18
  from udata.core.organization.models import Organization
19
+ from udata.core.pages.models import Page
18
20
  from udata.harvest.models import HarvestJob
19
21
  from udata.models import Activity, Discussion, Follow, TopicElement, Transfer, db
20
22
  from udata.storage.s3 import store_bytes
@@ -52,8 +54,18 @@ def purge_datasets(self):
52
54
  datasets = dataservice.datasets
53
55
  datasets.remove(dataset)
54
56
  dataservice.update(datasets=datasets)
55
- # Remove HarvestItem references
56
- HarvestJob.objects(items__dataset=dataset).update(set__items__S__dataset=None)
57
+ # Remove HarvestItem references (using update_many with array_filters to update all matching items)
58
+ HarvestJob._get_collection().update_many(
59
+ {"items.dataset": dataset.id},
60
+ {"$set": {"items.$[item].dataset": None}},
61
+ array_filters=[{"item.dataset": dataset.id}],
62
+ )
63
+ # Remove datasets in pages (mongoengine doesn't support updating a field in a generic embed)
64
+ Page._get_collection().update_many(
65
+ {"blocs.datasets": dataset.id},
66
+ {"$pull": {"blocs.$[b].datasets": dataset.id}},
67
+ array_filters=[{"b.datasets": dataset.id}],
68
+ )
57
69
  # Remove associated Transfers
58
70
  Transfer.objects(subject=dataset).delete()
59
71
  # Remove each dataset's resource's file
@@ -87,8 +99,7 @@ def get_queryset(model_cls):
87
99
  for attr in attrs:
88
100
  if getattr(model_cls, attr, None):
89
101
  params[attr] = False
90
- # no_cache to avoid eating up too much RAM
91
- return model_cls.objects.filter(**params).no_cache()
102
+ return model_cls.objects.filter(**params)
92
103
 
93
104
 
94
105
  def get_resource_for_csv_export_model(model, dataset):
@@ -166,7 +177,12 @@ def export_csv_for_model(model, dataset, replace: bool = False):
166
177
  dataset.save()
167
178
  # remove previous catalog if exists and replace is True
168
179
  if replace and fs_filename_to_remove:
169
- storages.resources.delete(fs_filename_to_remove)
180
+ try:
181
+ storages.resources.delete(fs_filename_to_remove)
182
+ except FileNotFoundError:
183
+ log.error(
184
+ f"File not found while deleting resource #{resource.id} ({fs_filename_to_remove}) in export_csv_for_model cleanup"
185
+ )
170
186
  return resource
171
187
  finally:
172
188
  csvfile.close()
@@ -210,8 +226,8 @@ def export_csv(self, model=None):
210
226
  with storages.resources.open(resource.fs_filename, "rb") as f:
211
227
  store_bytes(
212
228
  bucket=current_app.config["EXPORT_CSV_ARCHIVE_S3_BUCKET"],
213
- filename=f"{current_app.config['EXPORT_CSV_ARCHIVE_S3_FILENAME_PREFIX']}{resource.title}",
214
- bytes=f.read(),
229
+ filename=f"{current_app.config['EXPORT_CSV_ARCHIVE_S3_FILENAME_PREFIX']}{resource.title}.gz",
230
+ bytes=gzip.compress(f.read()),
215
231
  )
216
232
 
217
233
 
@@ -7,6 +7,7 @@ from flask_security import current_user
7
7
  from udata.api import API, api, fields
8
8
  from udata.core.dataservices.models import Dataservice
9
9
  from udata.core.dataset.models import Dataset
10
+ from udata.core.legal.mails import add_send_legal_notice_argument, send_legal_notice_on_deletion
10
11
  from udata.core.organization.api_fields import org_ref_fields
11
12
  from udata.core.organization.models import Organization
12
13
  from udata.core.reuse.models import Reuse
@@ -164,6 +165,9 @@ class DiscussionSpamAPI(SpamAPIMixin):
164
165
  model = Discussion
165
166
 
166
167
 
168
+ discussion_delete_parser = add_send_legal_notice_argument(api.parser())
169
+
170
+
167
171
  @ns.route("/<id>/", endpoint="discussion")
168
172
  class DiscussionAPI(API):
169
173
  """
@@ -236,11 +240,14 @@ class DiscussionAPI(API):
236
240
  return discussion
237
241
 
238
242
  @api.doc("delete_discussion")
243
+ @api.expect(discussion_delete_parser)
239
244
  @api.response(403, "Not allowed to delete this discussion")
240
245
  def delete(self, id):
241
246
  """Delete a discussion given its ID"""
247
+ args = discussion_delete_parser.parse_args()
242
248
  discussion = Discussion.objects.get_or_404(id=id_or_404(id))
243
249
  discussion.permissions["delete"].test()
250
+ send_legal_notice_on_deletion(discussion, args)
244
251
 
245
252
  discussion.delete()
246
253
  on_discussion_deleted.send(discussion)
@@ -259,6 +266,9 @@ class DiscussionCommentSpamAPI(SpamAPIMixin):
259
266
  return discussion, discussion.discussion[cidx]
260
267
 
261
268
 
269
+ message_delete_parser = add_send_legal_notice_argument(api.parser())
270
+
271
+
262
272
  @ns.route("/<id>/comments/<int:cidx>/", endpoint="discussion_comment")
263
273
  class DiscussionCommentAPI(API):
264
274
  """
@@ -286,16 +296,20 @@ class DiscussionCommentAPI(API):
286
296
  return discussion
287
297
 
288
298
  @api.doc("delete_discussion_comment")
299
+ @api.expect(message_delete_parser)
289
300
  @api.response(403, "Not allowed to delete this comment")
290
301
  def delete(self, id, cidx):
291
302
  """Delete a comment given its index"""
303
+ args = message_delete_parser.parse_args()
292
304
  discussion = Discussion.objects.get_or_404(id=id_or_404(id))
293
305
  if len(discussion.discussion) <= cidx:
294
306
  api.abort(404, "Comment does not exist")
295
307
  elif cidx == 0:
296
308
  api.abort(400, "You cannot delete the first comment of a discussion")
297
309
 
298
- discussion.discussion[cidx].permissions["delete"].test()
310
+ message = discussion.discussion[cidx]
311
+ message.permissions["delete"].test()
312
+ send_legal_notice_on_deletion(message, args)
299
313
 
300
314
  discussion.discussion.pop(cidx)
301
315
  discussion.save()
@@ -6,6 +6,7 @@ from flask_login import current_user
6
6
 
7
7
  from udata.core.linkable import Linkable
8
8
  from udata.core.spam.models import SpamMixin, spam_protected
9
+ from udata.i18n import lazy_gettext as _
9
10
  from udata.mongo import db
10
11
 
11
12
  from .signals import on_discussion_closed, on_new_discussion, on_new_discussion_comment
@@ -14,6 +15,9 @@ log = logging.getLogger(__name__)
14
15
 
15
16
 
16
17
  class Message(SpamMixin, db.EmbeddedDocument):
18
+ verbose_name = _("message")
19
+
20
+ id = db.AutoUUIDField()
17
21
  content = db.StringField(required=True)
18
22
  posted_on = db.DateTimeField(default=datetime.utcnow, required=True)
19
23
  posted_by = db.ReferenceField("User")
@@ -69,6 +73,8 @@ class Message(SpamMixin, db.EmbeddedDocument):
69
73
 
70
74
 
71
75
  class Discussion(SpamMixin, Linkable, db.Document):
76
+ verbose_name = _("discussion")
77
+
72
78
  user = db.ReferenceField("User")
73
79
  organization = db.ReferenceField("Organization")
74
80
 
File without changes
@@ -0,0 +1,128 @@
1
+ from flask import current_app
2
+ from flask_babel import LazyString
3
+ from flask_login import current_user
4
+ from flask_restx.inputs import boolean
5
+
6
+ from udata.core.dataservices.models import Dataservice
7
+ from udata.core.dataset.models import Dataset
8
+ from udata.core.discussions.models import Discussion, Message
9
+ from udata.core.organization.models import Organization
10
+ from udata.core.reuse.models import Reuse
11
+ from udata.core.user.models import User
12
+ from udata.i18n import lazy_gettext as _
13
+ from udata.mail import Link, MailMessage, ParagraphWithLinks
14
+
15
+ DeletableObject = Dataset | Reuse | Dataservice | Organization | User | Discussion | Message
16
+
17
+
18
+ def add_send_legal_notice_argument(parser):
19
+ """Add the send_legal_notice argument to a parser.
20
+
21
+ When send_legal_notice=true is passed by an admin, a formal legal notice email
22
+ is sent to the content owner. This email includes terms of use references and
23
+ information about how to contest the deletion (administrative appeal).
24
+ """
25
+ parser.add_argument(
26
+ "send_legal_notice",
27
+ type=boolean,
28
+ default=False,
29
+ location="args",
30
+ help="Send formal legal notice with appeal information to owner (admin only)",
31
+ )
32
+ return parser
33
+
34
+
35
+ def _get_recipients_for_organization(org: Organization) -> list[User]:
36
+ return [m.user for m in org.by_role("admin")]
37
+
38
+
39
+ def _get_recipients_for_owned_object(obj: Dataset | Reuse | Dataservice) -> list[User]:
40
+ if obj.owner:
41
+ return [obj.owner]
42
+ elif obj.organization:
43
+ return _get_recipients_for_organization(obj.organization)
44
+ return []
45
+
46
+
47
+ def send_legal_notice_on_deletion(obj: DeletableObject, args: dict):
48
+ """Send a formal legal notice email when content is deleted by an admin.
49
+
50
+ The email is only sent if:
51
+ - send_legal_notice=true was passed in args
52
+ - The current user is a sysadmin
53
+ """
54
+ if not args.get("send_legal_notice") or not current_user.sysadmin:
55
+ return
56
+
57
+ if isinstance(obj, Organization):
58
+ recipients = _get_recipients_for_organization(obj)
59
+ elif isinstance(obj, User):
60
+ recipients = [obj]
61
+ elif isinstance(obj, Discussion):
62
+ recipients = [obj.user] if obj.user else []
63
+ elif isinstance(obj, Message):
64
+ recipients = [obj.posted_by] if obj.posted_by else []
65
+ else:
66
+ recipients = _get_recipients_for_owned_object(obj)
67
+
68
+ if recipients:
69
+ _content_deleted(obj.verbose_name).send(recipients)
70
+
71
+
72
+ def _content_deleted(content_type_label: LazyString) -> MailMessage:
73
+ admin = current_user._get_current_object()
74
+ terms_of_use_url = current_app.config.get("TERMS_OF_USE_URL")
75
+ terms_of_use_deletion_article = current_app.config.get("TERMS_OF_USE_DELETION_ARTICLE")
76
+ telerecours_url = current_app.config.get("TELERECOURS_URL")
77
+
78
+ if terms_of_use_url and terms_of_use_deletion_article:
79
+ terms_paragraph = ParagraphWithLinks(
80
+ _(
81
+ 'Our %(terms_link)s specify in point %(article)s that the platform is not "intended '
82
+ "to disseminate advertising content, promotions of private interests, content contrary "
83
+ "to public order, illegal content, spam and any contribution violating the applicable "
84
+ "legal framework. The Editor reserves the right, without prior notice, to remove or "
85
+ "make inaccessible content published on the Platform that has no connection with its "
86
+ 'Purpose. The Editor does not carry out "a priori" control over publications. As soon '
87
+ "as the Editor becomes aware of content contrary to these terms of use, it acts quickly "
88
+ 'to remove or make it inaccessible".',
89
+ terms_link=Link(_("terms of use"), terms_of_use_url),
90
+ article=terms_of_use_deletion_article,
91
+ )
92
+ )
93
+ else:
94
+ terms_paragraph = _(
95
+ 'The platform is not "intended to disseminate advertising content, promotions of '
96
+ "private interests, content contrary to public order, illegal content, spam and any "
97
+ "contribution violating the applicable legal framework. The Editor reserves the right, "
98
+ "without prior notice, to remove or make inaccessible content published on the Platform "
99
+ 'that has no connection with its Purpose. The Editor does not carry out "a priori" '
100
+ "control over publications. As soon as the Editor becomes aware of content contrary to "
101
+ 'these terms of use, it acts quickly to remove or make it inaccessible".'
102
+ )
103
+
104
+ if telerecours_url:
105
+ appeal_paragraph = ParagraphWithLinks(
106
+ _(
107
+ "You may contest this decision within two months of its notification by filing "
108
+ "an administrative appeal (recours gracieux ou hiérarchique). You may also bring "
109
+ 'the matter before the administrative court via the "%(telerecours_link)s" application.',
110
+ telerecours_link=Link(_("Télérecours citoyens"), telerecours_url),
111
+ )
112
+ )
113
+ else:
114
+ appeal_paragraph = _("You may contest this decision by contacting us.")
115
+
116
+ paragraphs = [
117
+ _("Your %(content_type)s has been deleted.", content_type=content_type_label),
118
+ terms_paragraph,
119
+ appeal_paragraph,
120
+ _("Best regards,"),
121
+ admin.fullname,
122
+ _("%(site)s team member", site=current_app.config.get("SITE_TITLE", "data.gouv.fr")),
123
+ ]
124
+
125
+ return MailMessage(
126
+ subject=_("Deletion of your %(content_type)s", content_type=content_type_label),
127
+ paragraphs=paragraphs,
128
+ )
@@ -21,6 +21,7 @@ from udata.core.discussions.api import discussion_fields
21
21
  from udata.core.discussions.csv import DiscussionCsvAdapter
22
22
  from udata.core.discussions.models import Discussion
23
23
  from udata.core.followers.api import FollowAPI
24
+ from udata.core.legal.mails import add_send_legal_notice_argument, send_legal_notice_on_deletion
24
25
  from udata.core.reuse.models import Reuse
25
26
  from udata.core.storages.api import (
26
27
  image_parser,
@@ -137,6 +138,9 @@ class OrganizationListAPI(API):
137
138
  return organization, 201
138
139
 
139
140
 
141
+ org_delete_parser = add_send_legal_notice_argument(api.parser())
142
+
143
+
140
144
  @ns.route("/<org:org>/", endpoint="organization", doc=common_doc)
141
145
  @api.response(404, "Organization not found")
142
146
  @api.response(410, "Organization has been deleted")
@@ -170,12 +174,16 @@ class OrganizationAPI(API):
170
174
 
171
175
  @api.secure
172
176
  @api.doc("delete_organization")
177
+ @api.expect(org_delete_parser)
173
178
  @api.response(204, "Organization deleted")
174
179
  def delete(self, org):
175
180
  """Delete a organization given its identifier"""
181
+ args = org_delete_parser.parse_args()
176
182
  if org.deleted:
177
183
  api.abort(410, "Organization has been deleted")
178
184
  EditOrganizationPermission(org).test()
185
+ send_legal_notice_on_deletion(org, args)
186
+
179
187
  org.deleted = datetime.utcnow()
180
188
  org.save()
181
189
  return "", 204
@@ -383,12 +391,13 @@ class MembershipRequestAPI(API):
383
391
 
384
392
  form = api.validate(MembershipRequestForm, membership_request)
385
393
 
386
- if not membership_request:
394
+ if membership_request:
395
+ form.populate_obj(membership_request)
396
+ org.save()
397
+ else:
387
398
  membership_request = MembershipRequest()
388
- org.requests.append(membership_request)
389
-
390
- form.populate_obj(membership_request)
391
- org.save()
399
+ form.populate_obj(membership_request)
400
+ org.add_membership_request(membership_request)
392
401
 
393
402
  notify_membership_request.delay(str(org.id), str(membership_request.id))
394
403
 
@@ -424,6 +433,7 @@ class MembershipAcceptAPI(MembershipAPI):
424
433
  org.members.append(member)
425
434
  org.count_members()
426
435
  org.save()
436
+ MembershipRequest.after_handle.send(membership_request, org=org)
427
437
 
428
438
  notify_membership_response.delay(str(org.id), str(membership_request.id))
429
439
 
@@ -446,6 +456,7 @@ class MembershipRefuseAPI(MembershipAPI):
446
456
  membership_request.refusal_comment = form.comment.data
447
457
 
448
458
  org.save()
459
+ MembershipRequest.after_handle.send(membership_request, org=org)
449
460
 
450
461
  notify_membership_response.delay(str(org.id), str(membership_request.id))
451
462
 
@@ -3,7 +3,6 @@ from flask import request
3
3
  from udata import search
4
4
  from udata.api import API, apiv2
5
5
  from udata.core.contact_point.api_fields import contact_point_fields
6
- from udata.utils import multi_to_dict
7
6
 
8
7
  from .api_fields import member_fields, org_fields, org_page_fields
9
8
  from .permissions import EditOrganizationPermission
@@ -30,8 +29,8 @@ class OrganizationSearchAPI(API):
30
29
  @apiv2.marshal_with(org_page_fields)
31
30
  def get(self):
32
31
  """Search all organizations"""
33
- search_parser.parse_args()
34
- return search.query(OrganizationSearch, **multi_to_dict(request.args))
32
+ args = search_parser.parse_args()
33
+ return search.query(OrganizationSearch, **args)
35
34
 
36
35
 
37
36
  @ns.route("/<org:org>/extras/", endpoint="organization_extras")
@@ -16,7 +16,7 @@ def new_membership_request(org: Organization, request: MembershipRequest) -> Mai
16
16
  )
17
17
  ),
18
18
  LabelledContent(_("Reason for the request:"), request.comment),
19
- MailCTA(_("See the request"), cdata_url(f"/admin/organizations/{org.id}/members/")),
19
+ MailCTA(_("See the request"), cdata_url(f"/admin/organizations/{org.id}/members")),
20
20
  ],
21
21
  )
22
22
 
@@ -81,6 +81,9 @@ class MembershipRequest(db.EmbeddedDocument):
81
81
  comment = db.StringField()
82
82
  refusal_comment = db.StringField()
83
83
 
84
+ after_create = Signal()
85
+ after_handle = Signal()
86
+
84
87
  @property
85
88
  def status_label(self):
86
89
  return MEMBERSHIP_STATUS[self.status]
@@ -123,7 +126,10 @@ class Organization(
123
126
  db.SlugField(max_length=255, required=True, populate_from="name", update=True, follow=True),
124
127
  auditable=False,
125
128
  )
126
- description = field(db.StringField(required=True))
129
+ description = field(
130
+ db.StringField(required=True),
131
+ markdown=True,
132
+ )
127
133
  url = field(db.URLField())
128
134
  image_url = field(db.StringField())
129
135
  logo = field(
@@ -162,6 +168,8 @@ class Organization(
162
168
  "auto_create_index_on_save": True,
163
169
  }
164
170
 
171
+ verbose_name = _("organization")
172
+
165
173
  def __str__(self):
166
174
  return self.name or ""
167
175
 
@@ -198,7 +206,7 @@ class Organization(
198
206
  cls.before_save.send(document)
199
207
 
200
208
  def self_web_url(self, **kwargs):
201
- return cdata_url(f"/organizations/{self._link_id(**kwargs)}/", **kwargs)
209
+ return cdata_url(f"/organizations/{self._link_id(**kwargs)}", **kwargs)
202
210
 
203
211
  def self_api_url(self, **kwargs):
204
212
  return url_for(
@@ -304,6 +312,11 @@ class Organization(
304
312
  def views_count(self):
305
313
  return self.metrics.get("views", 0)
306
314
 
315
+ def add_membership_request(self, membership_request):
316
+ self.requests.append(membership_request)
317
+ self.save()
318
+ MembershipRequest.after_create.send(membership_request, org=self)
319
+
307
320
  def count_members(self):
308
321
  self.metrics["members"] = len(self.members)
309
322
  self.save(signal_kwargs={"ignores": ["post_save"]})