udata 14.0.3.dev1__py3-none-any.whl → 14.7.3.dev4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (151) hide show
  1. udata/api/__init__.py +2 -0
  2. udata/api_fields.py +120 -19
  3. udata/app.py +18 -20
  4. udata/auth/__init__.py +4 -7
  5. udata/auth/forms.py +3 -3
  6. udata/auth/views.py +13 -6
  7. udata/commands/dcat.py +1 -1
  8. udata/commands/serve.py +3 -11
  9. udata/core/activity/api.py +5 -6
  10. udata/core/badges/tests/test_tasks.py +0 -2
  11. udata/core/csv.py +5 -0
  12. udata/core/dataservices/api.py +8 -1
  13. udata/core/dataservices/apiv2.py +3 -6
  14. udata/core/dataservices/models.py +5 -2
  15. udata/core/dataservices/rdf.py +2 -1
  16. udata/core/dataservices/tasks.py +6 -2
  17. udata/core/dataset/api.py +30 -4
  18. udata/core/dataset/api_fields.py +1 -1
  19. udata/core/dataset/apiv2.py +1 -1
  20. udata/core/dataset/constants.py +2 -9
  21. udata/core/dataset/models.py +21 -9
  22. udata/core/dataset/permissions.py +31 -0
  23. udata/core/dataset/rdf.py +18 -16
  24. udata/core/dataset/tasks.py +16 -7
  25. udata/core/discussions/api.py +15 -1
  26. udata/core/discussions/models.py +6 -0
  27. udata/core/legal/__init__.py +0 -0
  28. udata/core/legal/mails.py +128 -0
  29. udata/core/organization/api.py +16 -5
  30. udata/core/organization/api_fields.py +3 -3
  31. udata/core/organization/apiv2.py +3 -4
  32. udata/core/organization/mails.py +1 -1
  33. udata/core/organization/models.py +40 -7
  34. udata/core/organization/notifications.py +84 -0
  35. udata/core/organization/permissions.py +1 -1
  36. udata/core/organization/tasks.py +3 -0
  37. udata/core/pages/models.py +49 -0
  38. udata/core/pages/tests/test_api.py +165 -1
  39. udata/core/post/api.py +25 -70
  40. udata/core/post/constants.py +8 -0
  41. udata/core/post/models.py +109 -17
  42. udata/core/post/tests/test_api.py +140 -3
  43. udata/core/post/tests/test_models.py +24 -0
  44. udata/core/reports/api.py +18 -0
  45. udata/core/reports/models.py +42 -2
  46. udata/core/reuse/api.py +8 -0
  47. udata/core/reuse/apiv2.py +3 -6
  48. udata/core/reuse/models.py +1 -1
  49. udata/core/spatial/forms.py +2 -2
  50. udata/core/topic/models.py +8 -2
  51. udata/core/user/api.py +10 -3
  52. udata/core/user/api_fields.py +3 -3
  53. udata/core/user/models.py +33 -8
  54. udata/features/notifications/api.py +7 -18
  55. udata/features/notifications/models.py +59 -0
  56. udata/features/notifications/tasks.py +25 -0
  57. udata/features/transfer/actions.py +2 -0
  58. udata/features/transfer/models.py +17 -0
  59. udata/features/transfer/notifications.py +96 -0
  60. udata/flask_mongoengine/engine.py +0 -4
  61. udata/flask_mongoengine/pagination.py +1 -1
  62. udata/frontend/markdown.py +2 -1
  63. udata/harvest/actions.py +20 -0
  64. udata/harvest/api.py +24 -7
  65. udata/harvest/backends/base.py +27 -1
  66. udata/harvest/backends/ckan/harvesters.py +21 -4
  67. udata/harvest/backends/dcat.py +4 -1
  68. udata/harvest/commands.py +33 -0
  69. udata/harvest/filters.py +17 -6
  70. udata/harvest/models.py +16 -0
  71. udata/harvest/permissions.py +27 -0
  72. udata/harvest/tests/ckan/test_ckan_backend.py +33 -0
  73. udata/harvest/tests/test_actions.py +46 -2
  74. udata/harvest/tests/test_api.py +161 -6
  75. udata/harvest/tests/test_base_backend.py +86 -1
  76. udata/harvest/tests/test_dcat_backend.py +68 -3
  77. udata/harvest/tests/test_filters.py +6 -0
  78. udata/i18n.py +1 -4
  79. udata/mail.py +14 -0
  80. udata/migrations/2021-08-17-harvest-integrity.py +23 -16
  81. udata/migrations/2025-10-31-create-membership-request-notifications.py +55 -0
  82. udata/migrations/2025-12-04-add-uuid-to-discussion-messages.py +28 -0
  83. udata/migrations/2025-12-16-create-transfer-request-notifications.py +69 -0
  84. udata/migrations/2026-01-14-add-default-kind-to-posts.py +17 -0
  85. udata/mongo/slug_fields.py +1 -1
  86. udata/rdf.py +65 -11
  87. udata/routing.py +2 -2
  88. udata/settings.py +11 -0
  89. udata/tasks.py +2 -0
  90. udata/templates/mail/message.html +3 -1
  91. udata/tests/api/__init__.py +7 -17
  92. udata/tests/api/test_activities_api.py +36 -0
  93. udata/tests/api/test_datasets_api.py +69 -0
  94. udata/tests/api/test_organizations_api.py +0 -3
  95. udata/tests/api/test_reports_api.py +157 -0
  96. udata/tests/api/test_user_api.py +1 -1
  97. udata/tests/apiv2/test_dataservices.py +14 -0
  98. udata/tests/apiv2/test_organizations.py +9 -0
  99. udata/tests/apiv2/test_reuses.py +11 -0
  100. udata/tests/cli/test_cli_base.py +0 -1
  101. udata/tests/dataservice/test_dataservice_tasks.py +29 -0
  102. udata/tests/dataset/test_dataset_model.py +13 -1
  103. udata/tests/dataset/test_dataset_rdf.py +164 -5
  104. udata/tests/dataset/test_dataset_tasks.py +25 -0
  105. udata/tests/frontend/test_auth.py +58 -1
  106. udata/tests/frontend/test_csv.py +0 -3
  107. udata/tests/helpers.py +31 -27
  108. udata/tests/organization/test_notifications.py +67 -2
  109. udata/tests/search/test_search_integration.py +70 -0
  110. udata/tests/site/test_site_csv_exports.py +22 -10
  111. udata/tests/test_activity.py +9 -9
  112. udata/tests/test_api_fields.py +10 -0
  113. udata/tests/test_discussions.py +5 -5
  114. udata/tests/test_legal_mails.py +359 -0
  115. udata/tests/test_notifications.py +15 -57
  116. udata/tests/test_notifications_task.py +43 -0
  117. udata/tests/test_owned.py +81 -1
  118. udata/tests/test_transfer.py +181 -2
  119. udata/tests/test_uris.py +33 -0
  120. udata/translations/ar/LC_MESSAGES/udata.mo +0 -0
  121. udata/translations/ar/LC_MESSAGES/udata.po +309 -158
  122. udata/translations/de/LC_MESSAGES/udata.mo +0 -0
  123. udata/translations/de/LC_MESSAGES/udata.po +313 -160
  124. udata/translations/es/LC_MESSAGES/udata.mo +0 -0
  125. udata/translations/es/LC_MESSAGES/udata.po +312 -160
  126. udata/translations/fr/LC_MESSAGES/udata.mo +0 -0
  127. udata/translations/fr/LC_MESSAGES/udata.po +475 -202
  128. udata/translations/it/LC_MESSAGES/udata.mo +0 -0
  129. udata/translations/it/LC_MESSAGES/udata.po +317 -162
  130. udata/translations/pt/LC_MESSAGES/udata.mo +0 -0
  131. udata/translations/pt/LC_MESSAGES/udata.po +315 -161
  132. udata/translations/sr/LC_MESSAGES/udata.mo +0 -0
  133. udata/translations/sr/LC_MESSAGES/udata.po +323 -164
  134. udata/translations/udata.pot +169 -124
  135. udata/uris.py +0 -2
  136. udata/utils.py +23 -0
  137. udata-14.7.3.dev4.dist-info/METADATA +109 -0
  138. {udata-14.0.3.dev1.dist-info → udata-14.7.3.dev4.dist-info}/RECORD +142 -135
  139. udata/core/post/forms.py +0 -30
  140. udata/flask_mongoengine/json.py +0 -38
  141. udata/templates/mail/base.html +0 -105
  142. udata/templates/mail/base.txt +0 -6
  143. udata/templates/mail/button.html +0 -3
  144. udata/templates/mail/layouts/1-column.html +0 -19
  145. udata/templates/mail/layouts/2-columns.html +0 -20
  146. udata/templates/mail/layouts/center-panel.html +0 -16
  147. udata-14.0.3.dev1.dist-info/METADATA +0 -132
  148. {udata-14.0.3.dev1.dist-info → udata-14.7.3.dev4.dist-info}/WHEEL +0 -0
  149. {udata-14.0.3.dev1.dist-info → udata-14.7.3.dev4.dist-info}/entry_points.txt +0 -0
  150. {udata-14.0.3.dev1.dist-info → udata-14.7.3.dev4.dist-info}/licenses/LICENSE +0 -0
  151. {udata-14.0.3.dev1.dist-info → udata-14.7.3.dev4.dist-info}/top_level.txt +0 -0
udata/api/__init__.py CHANGED
@@ -121,6 +121,8 @@ class UDataApi(Api):
121
121
  if "application/json" not in request.headers.get("Content-Type", ""):
122
122
  errors = {"Content-Type": "expecting application/json"}
123
123
  self.abort(400, errors=errors)
124
+ if not isinstance(request.json, dict):
125
+ self.abort(400, errors={"request": "expecting a JSON object"})
124
126
  form = form_cls.from_json(request.json, obj=obj, instance=obj, meta={"csrf": False})
125
127
  if not form.validate():
126
128
  self.abort(400, errors=form.errors)
udata/api_fields.py CHANGED
@@ -29,7 +29,7 @@ import flask_restx.fields as restx_fields
29
29
  import mongoengine
30
30
  import mongoengine.fields as mongo_fields
31
31
  from bson import DBRef, ObjectId
32
- from flask import Request
32
+ from flask import Request, request
33
33
  from flask_restx import marshal
34
34
  from flask_restx.inputs import boolean
35
35
  from flask_restx.reqparse import RequestParser
@@ -40,6 +40,32 @@ from udata.api import api, base_reference
40
40
  from udata.mongo.errors import FieldValidationError
41
41
  from udata.mongo.queryset import DBPaginator, UDataQuerySet
42
42
 
43
+
44
+ def required_if(**conditions):
45
+ """Check helper that makes a field required when other fields have specific values.
46
+
47
+ Usage:
48
+ page_id = field(
49
+ db.ReferenceField("Page"),
50
+ checks=[required_if(body_type="blocs")],
51
+ )
52
+ """
53
+
54
+ def check(value, data, field, obj, **_kwargs):
55
+ if value is not None:
56
+ return
57
+ for condition_field, condition_value in conditions.items():
58
+ actual_condition = data.get(condition_field, getattr(obj, condition_field, None))
59
+ if actual_condition == condition_value:
60
+ raise FieldValidationError(
61
+ f"'{field}' is required when '{condition_field}' is '{condition_value}'",
62
+ field=field,
63
+ )
64
+
65
+ check.run_even_if_missing = True
66
+ return check
67
+
68
+
43
69
  lazy_reference = api.model(
44
70
  "LazyReference",
45
71
  {
@@ -55,8 +81,8 @@ classes_by_parents = {}
55
81
 
56
82
 
57
83
  class GenericField(restx_fields.Raw):
58
- def __init__(self, fields_by_type):
59
- super().__init__(self)
84
+ def __init__(self, fields_by_type, **kwargs):
85
+ super(GenericField, self).__init__(**kwargs)
60
86
  self.default = None
61
87
  self.fields_by_type = fields_by_type
62
88
 
@@ -79,13 +105,15 @@ def convert_db_to_field(key, field, info) -> tuple[Callable | None, Callable | N
79
105
  user-supplied overrides, setting the readonly flag…), it's easier to have to do this only once at the end of the function.
80
106
 
81
107
  """
108
+ from udata.mongo.engine import db
109
+
82
110
  params: dict = {}
83
111
  params["required"] = field.required
84
112
 
85
113
  read_params: dict = {}
86
114
  write_params: dict = {}
87
115
 
88
- constructor: Callable
116
+ constructor: Callable | None = None
89
117
  constructor_read: Callable | None = None
90
118
  constructor_write: Callable | None = None
91
119
 
@@ -204,13 +232,34 @@ def convert_db_to_field(key, field, info) -> tuple[Callable | None, Callable | N
204
232
  def constructor_write(**kwargs):
205
233
  return restx_fields.List(field_write, **kwargs)
206
234
 
207
- elif isinstance(
208
- field, (mongo_fields.GenericReferenceField, mongoengine.fields.GenericLazyReferenceField)
209
- ):
235
+ elif isinstance(field, mongoengine.fields.GenericLazyReferenceField):
210
236
 
211
237
  def constructor(**kwargs):
212
238
  return restx_fields.Nested(lazy_reference, **kwargs)
213
239
 
240
+ elif isinstance(field, mongo_fields.GenericReferenceField):
241
+ if field.choices:
242
+ generic_fields = {}
243
+ for cls in field.choices:
244
+ cls = db.resolve_model(cls) if isinstance(cls, str) else cls
245
+ generic_fields[cls.__name__] = convert_db_to_field(
246
+ f"{key}.{cls.__name__}",
247
+ # Instead of having GenericReferenceField() we'll create fields for each
248
+ # of the subclasses with ReferenceField(Organization)…
249
+ mongoengine.fields.ReferenceField(cls),
250
+ info,
251
+ )
252
+
253
+ def constructor_read(**kwargs):
254
+ return GenericField({k: v[0].model for k, v in generic_fields.items()}, **kwargs)
255
+
256
+ def constructor_write(**kwargs):
257
+ return GenericField({k: v[1].model for k, v in generic_fields.items()}, **kwargs)
258
+ else:
259
+
260
+ def constructor(**kwargs):
261
+ return restx_fields.Nested(lazy_reference, **kwargs)
262
+
214
263
  elif isinstance(field, mongo_fields.ReferenceField | mongo_fields.LazyReferenceField):
215
264
  # For reference we accept while writing a String representing the ID of the referenced model.
216
265
  # For reading, if the user supplied a `nested_fields` (RestX model), we use it to convert
@@ -230,6 +279,23 @@ def convert_db_to_field(key, field, info) -> tuple[Callable | None, Callable | N
230
279
 
231
280
  write_params["description"] = "ID of the reference"
232
281
  constructor_write = restx_fields.String
282
+ elif isinstance(field, mongo_fields.GenericEmbeddedDocumentField):
283
+ generic_fields = {
284
+ cls.__name__: convert_db_to_field(
285
+ f"{key}.{cls.__name__}",
286
+ # Instead of having GenericEmbeddedDocumentField() we'll create fields for each
287
+ # of the subclasses with EmbededdDocumentField(MembershipRequestNotificationDetails)…
288
+ mongoengine.fields.EmbeddedDocumentField(cls),
289
+ info,
290
+ )
291
+ for cls in field.choices
292
+ }
293
+
294
+ def constructor_read(**kwargs):
295
+ return GenericField({k: v[0].model for k, v in generic_fields.items()}, **kwargs)
296
+
297
+ def constructor_write(**kwargs):
298
+ return GenericField({k: v[1].model for k, v in generic_fields.items()}, **kwargs)
233
299
  elif isinstance(field, mongo_fields.EmbeddedDocumentField):
234
300
  nested_fields = info.get("nested_fields")
235
301
  if nested_fields is not None:
@@ -322,12 +388,12 @@ def generate_fields(**kwargs) -> Callable:
322
388
  write_fields: dict = {}
323
389
  ref_fields: dict = {}
324
390
  sortables: list = kwargs.get("additional_sorts", [])
391
+ default_sort: list = kwargs.get("default_sort", None)
325
392
 
326
393
  filterables: list[dict] = kwargs.get("standalone_filters", [])
327
394
  nested_filters: dict[str, dict] = get_fields_with_nested_filters(
328
395
  kwargs.get("nested_filters", {})
329
396
  )
330
-
331
397
  if issubclass(cls, db.Document) or issubclass(cls, db.DynamicDocument):
332
398
  read_fields["id"] = restx_fields.String(required=True, readonly=True)
333
399
 
@@ -472,6 +538,7 @@ def generate_fields(**kwargs) -> Callable:
472
538
  type=str,
473
539
  location="args",
474
540
  choices=choices,
541
+ default=default_sort,
475
542
  help="The field (and direction) on which sorting apply",
476
543
  )
477
544
 
@@ -511,6 +578,9 @@ def generate_fields(**kwargs) -> Callable:
511
578
  phrase_query: str = " ".join([f'"{elem}"' for elem in args["q"].split(" ")])
512
579
  base_query = base_query.search_text(phrase_query)
513
580
 
581
+ if "sort" not in request.args:
582
+ base_query = base_query.order_by("$text_score")
583
+
514
584
  for filterable in filterables:
515
585
  # If it's from an `nested_filter`, use the custom label instead of the key,
516
586
  # eg use `organization_badge` instead of `organization.badges` which is
@@ -637,6 +707,19 @@ def field(
637
707
  return inner
638
708
 
639
709
 
710
+ def run_check(check, value, key, obj, data):
711
+ check(
712
+ value,
713
+ **{
714
+ "is_creation": obj._created,
715
+ "is_update": not obj._created,
716
+ "field": key,
717
+ "obj": obj,
718
+ "data": data,
719
+ },
720
+ )
721
+
722
+
640
723
  def patch(obj, request) -> type:
641
724
  """Patch the object with the data from the request.
642
725
 
@@ -646,12 +729,16 @@ def patch(obj, request) -> type:
646
729
  from udata.mongo.engine import db
647
730
 
648
731
  data = request.json if isinstance(request, Request) else request
732
+
649
733
  for key, value in data.items():
650
734
  field = obj.__write_fields__.get(key)
651
735
  if field is not None and not field.readonly:
652
736
  model_attribute = getattr(obj.__class__, key)
653
737
  info = getattr(model_attribute, "__additional_field_info__", {})
654
738
 
739
+ if value == "" and isinstance(model_attribute, mongo_fields.StringField):
740
+ value = None
741
+
655
742
  if hasattr(model_attribute, "from_input"):
656
743
  value = model_attribute.from_input(value)
657
744
  elif isinstance(model_attribute, mongoengine.fields.ListField) and isinstance(
@@ -677,6 +764,13 @@ def patch(obj, request) -> type:
677
764
  value["id"],
678
765
  document_type=db.resolve_model(value["class"]),
679
766
  )
767
+ elif value and isinstance(
768
+ model_attribute,
769
+ mongoengine.fields.GenericEmbeddedDocumentField,
770
+ ):
771
+ generic_key = info.get("generic_key", DEFAULT_GENERIC_KEY)
772
+ embedded_field = classes_by_names[value[generic_key]]
773
+ value = patch(embedded_field(), value)
680
774
  elif value and isinstance(
681
775
  model_attribute,
682
776
  mongoengine.fields.EmbeddedDocumentField,
@@ -703,23 +797,30 @@ def patch(obj, request) -> type:
703
797
 
704
798
  value = objects
705
799
 
706
- # `checks` field attribute allows to do validation from the request before setting
707
- # the attribute
800
+ # Run checks if value is modified.
801
+ # We run checks here (before setattr) to compare old vs new value.
708
802
  checks = info.get("checks", [])
709
-
710
803
  if is_value_modified(getattr(obj, key), value):
711
804
  for check in checks:
712
- check(
713
- value,
714
- **{
715
- "is_creation": obj._created,
716
- "is_update": not obj._created,
717
- "field": key,
718
- },
719
- ) # TODO add other model attributes in function parameters
805
+ run_check(check, value, key, obj, data)
720
806
 
721
807
  setattr(obj, key, value)
722
808
 
809
+ # Run checks marked with `run_even_if_missing` on fields not in request.
810
+ # Some checks (like `required_if`) need to run even when their field is absent
811
+ # from the request, because they validate cross-field constraints based on
812
+ # other fields in the request (e.g. "page_id is required if body_type is blocs").
813
+ for key, _, info in get_fields(obj.__class__):
814
+ if key in data:
815
+ continue
816
+ checks = info.get("checks", [])
817
+ value = getattr(obj, key, None)
818
+
819
+ for check in checks:
820
+ if not getattr(check, "run_even_if_missing", False):
821
+ continue
822
+ run_check(check, value, key, obj, data)
823
+
723
824
  return obj
724
825
 
725
826
 
udata/app.py CHANGED
@@ -1,26 +1,28 @@
1
- import datetime
2
1
  import logging
3
2
  import os
4
3
  import types
4
+ from datetime import datetime
5
5
  from importlib.metadata import entry_points
6
6
  from os.path import abspath, dirname, exists, isfile, join
7
7
 
8
8
  import bson
9
+ from bson import json_util
9
10
  from flask import Blueprint as BaseBlueprint
10
11
  from flask import (
11
12
  Flask,
12
13
  abort,
13
14
  g,
14
- json,
15
15
  jsonify,
16
16
  make_response,
17
17
  render_template,
18
18
  request,
19
19
  send_from_directory,
20
20
  )
21
+ from flask.json.provider import DefaultJSONProvider
21
22
  from flask_caching import Cache
22
23
  from flask_wtf.csrf import CSRFProtect
23
- from speaklater import is_lazy_string
24
+ from mongoengine import EmbeddedDocument
25
+ from mongoengine.base import BaseDocument
24
26
  from werkzeug.exceptions import NotFound
25
27
  from werkzeug.middleware.proxy_fix import ProxyFix
26
28
 
@@ -108,9 +110,9 @@ class Blueprint(BaseBlueprint):
108
110
  return wrapper
109
111
 
110
112
 
111
- class UDataJsonEncoder(json.JSONEncoder):
113
+ class UdataJsonProvider(DefaultJSONProvider):
112
114
  """
113
- A JSONEncoder subclass to encode unsupported types:
115
+ A JSONProvider subclass to encode unsupported types:
114
116
 
115
117
  - ObjectId
116
118
  - datetime
@@ -120,21 +122,16 @@ class UDataJsonEncoder(json.JSONEncoder):
120
122
  Ensure an app context is always present.
121
123
  """
122
124
 
123
- def default(self, obj):
124
- if is_lazy_string(obj):
125
+ @staticmethod
126
+ def default(obj):
127
+ if isinstance(obj, BaseDocument) or isinstance(obj, EmbeddedDocument):
128
+ return json_util._json_convert(obj.to_mongo())
129
+ elif isinstance(obj, bson.ObjectId):
125
130
  return str(obj)
126
- elif isinstance(obj, bson.objectid.ObjectId):
127
- return str(obj)
128
- elif isinstance(obj, datetime.datetime):
131
+ elif isinstance(obj, datetime):
129
132
  return obj.isoformat()
130
- elif hasattr(obj, "to_dict"):
131
- return obj.to_dict()
132
- elif hasattr(obj, "serialize"):
133
- return obj.serialize()
134
- # Serialize Raw data for Document and EmbeddedDocument.
135
- elif hasattr(obj, "_data"):
136
- return obj._data
137
- return super(UDataJsonEncoder, self).default(obj)
133
+
134
+ return super(UdataJsonProvider, UdataJsonProvider).default(obj)
138
135
 
139
136
 
140
137
  # These loggers are very verbose
@@ -166,10 +163,11 @@ def create_app(config="udata.settings.Defaults", override=None, init_logging=ini
166
163
  if override:
167
164
  app.config.from_object(override)
168
165
 
169
- app.json_encoder = UDataJsonEncoder
166
+ app.json_provider_class = UdataJsonProvider
167
+ app.json = app.json_provider_class(app)
170
168
 
171
169
  # `ujson` doesn't support `cls` parameter https://github.com/ultrajson/ultrajson/issues/124
172
- app.config["RESTX_JSON"] = {"cls": UDataJsonEncoder}
170
+ app.config["RESTX_JSON"] = {"default": UdataJsonProvider.default}
173
171
 
174
172
  app.debug = app.config["DEBUG"] and not app.config["TESTING"]
175
173
 
udata/auth/__init__.py CHANGED
@@ -26,9 +26,6 @@ def render_security_template(template_name_or_list, **kwargs):
26
26
  return render_template(template_name_or_list, **kwargs)
27
27
 
28
28
 
29
- security = Security()
30
-
31
-
32
29
  class Permission(BasePermission):
33
30
  def __init__(self, *needs):
34
31
  """Let administrator bypass all permissions"""
@@ -84,15 +81,13 @@ def init_app(app):
84
81
  # Same logic as in our own mail system :DisableMail
85
82
  debug = app.config.get("DEBUG", False)
86
83
  send_mail = app.config.get("SEND_MAIL", not debug)
87
- mail_util_cls = None if send_mail else NoopMailUtil
84
+ mail_util_cls = mail_util.MailUtil if send_mail else NoopMailUtil
88
85
 
89
- security.init_app(
90
- app,
86
+ security = Security(
91
87
  datastore,
92
88
  register_blueprint=False,
93
89
  render_template=render_security_template,
94
90
  login_form=ExtendedLoginForm,
95
- confirm_register_form=ExtendedRegisterForm,
96
91
  register_form=ExtendedRegisterForm,
97
92
  reset_password_form=ExtendedResetPasswordForm,
98
93
  forgot_password_form=ExtendedForgotPasswordForm,
@@ -100,6 +95,8 @@ def init_app(app):
100
95
  mail_util_cls=mail_util_cls,
101
96
  )
102
97
 
98
+ security.init_app(app, datastore, register_blueprint=False)
99
+
103
100
  security_bp = create_security_blueprint(app, app.extensions["security"], "security_blueprint")
104
101
 
105
102
  app.register_blueprint(security_bp)
udata/auth/forms.py CHANGED
@@ -8,7 +8,7 @@ from flask_security.forms import (
8
8
  ForgotPasswordForm,
9
9
  Form,
10
10
  LoginForm,
11
- RegisterForm,
11
+ RegisterFormV2,
12
12
  ResetPasswordForm,
13
13
  )
14
14
 
@@ -31,7 +31,7 @@ class WithCaptcha:
31
31
  return False
32
32
 
33
33
 
34
- class ExtendedRegisterForm(WithCaptcha, RegisterForm):
34
+ class ExtendedRegisterForm(WithCaptcha, RegisterFormV2):
35
35
  first_name = fields.StringField(
36
36
  _("First name"),
37
37
  [
@@ -49,7 +49,7 @@ class ExtendedRegisterForm(WithCaptcha, RegisterForm):
49
49
  accept_conditions = fields.BooleanField(
50
50
  _("J'accepte les conditions générales d'utilisation"),
51
51
  validators=[
52
- validators.DataRequired(message=_("Vous devez accepter les CGU pour continuer."))
52
+ validators.DataRequired(message=_("You must accept the terms of use to continue."))
53
53
  ],
54
54
  )
55
55
 
udata/auth/views.py CHANGED
@@ -57,11 +57,15 @@ def confirm_change_email_token_status(token):
57
57
  token, "confirm", get_within_delta("CONFIRM_EMAIL_WITHIN")
58
58
  )
59
59
  new_email = None
60
+ user = None
60
61
 
61
62
  if not invalid and token_data:
62
- user, token_email_hash, new_email = token_data
63
- user = _datastore.find_user(fs_uniquifier=user)
64
- invalid = not verify_hash(token_email_hash, user.email)
63
+ user_uniquifier, token_email_hash, new_email = token_data
64
+ user = _datastore.find_user(fs_uniquifier=user_uniquifier)
65
+ if user is None:
66
+ invalid = True
67
+ else:
68
+ invalid = not verify_hash(token_email_hash, user.email)
65
69
 
66
70
  return expired, invalid, user, new_email
67
71
 
@@ -84,6 +88,11 @@ def confirm_change_email(token):
84
88
  if flash:
85
89
  return redirect(homepage_url(flash=flash, flash_data=flash_data))
86
90
 
91
+ # Check if the new email is already taken by another user
92
+ existing_user = _datastore.find_user(email=new_email)
93
+ if existing_user and existing_user.id != user.id:
94
+ return redirect(homepage_url(flash="change_email_already_taken"))
95
+
87
96
  if user != current_user:
88
97
  logout_user()
89
98
  login_user(user)
@@ -140,10 +149,8 @@ def create_security_blueprint(app, state, import_name):
140
149
  This creates an I18nBlueprint to use as a base.
141
150
  """
142
151
  bp = I18nBlueprint(
143
- state.blueprint_name,
152
+ "security",
144
153
  import_name,
145
- url_prefix=state.url_prefix,
146
- subdomain=state.subdomain,
147
154
  template_folder="templates",
148
155
  )
149
156
 
udata/commands/dcat.py CHANGED
@@ -85,7 +85,7 @@ def parse_url(url, csw, iso, quiet=False, rid=""):
85
85
  echo("Item kwargs: {}".format(yellow(item.kwargs)))
86
86
  node = backend.get_node_from_item(graph, item)
87
87
  dataset = MockDatasetFactory()
88
- dataset = dataset_from_rdf(graph, dataset, node=node)
88
+ dataset = dataset_from_rdf(graph, dataset, node=node, dryrun=True)
89
89
  echo("")
90
90
  echo(green("Dataset found!"))
91
91
  echo("Title: {}".format(yellow(dataset)))
udata/commands/serve.py CHANGED
@@ -3,7 +3,7 @@ import os
3
3
 
4
4
  import click
5
5
  from flask import current_app
6
- from flask.cli import DispatchingApp, pass_script_info
6
+ from flask.cli import pass_script_info
7
7
  from werkzeug.serving import run_simple
8
8
 
9
9
  from udata.commands import cli
@@ -26,17 +26,11 @@ log = logging.getLogger(__name__)
26
26
  default=None,
27
27
  help="Enable or disable the debugger. By default the debugger is active if debug is enabled.",
28
28
  )
29
- @click.option(
30
- "--eager-loading/--lazy-loader",
31
- default=None,
32
- help="Enable or disable eager loading. By default eager "
33
- "loading is enabled if the reloader is disabled.",
34
- )
35
29
  @click.option(
36
30
  "--with-threads/--without-threads", default=True, help="Enable or disable multithreading."
37
31
  )
38
32
  @pass_script_info
39
- def serve(info, host, port, reload, debugger, eager_loading, with_threads):
33
+ def serve(info, host, port, reload, debugger, with_threads):
40
34
  """
41
35
  Runs a local udata development server.
42
36
 
@@ -62,10 +56,8 @@ def serve(info, host, port, reload, debugger, eager_loading, with_threads):
62
56
  reload = bool(debug)
63
57
  if debugger is None:
64
58
  debugger = bool(debug)
65
- if eager_loading is None:
66
- eager_loading = not reload
67
59
 
68
- app = DispatchingApp(info.load_app, use_eager_loading=eager_loading)
60
+ app = info.load_app()
69
61
 
70
62
  settings = os.environ.get("UDATA_SETTINGS", os.path.join(os.getcwd(), "udata.cfg"))
71
63
  extra_files = [settings]
@@ -4,8 +4,9 @@ from bson import ObjectId
4
4
  from mongoengine.errors import DoesNotExist
5
5
 
6
6
  from udata.api import API, api, fields
7
- from udata.auth import current_user
7
+ from udata.core.dataset.permissions import OwnableReadPermission
8
8
  from udata.core.organization.api_fields import org_ref_fields
9
+ from udata.core.owned import Owned
9
10
  from udata.core.user.api_fields import user_ref_fields
10
11
  from udata.models import Activity, db
11
12
 
@@ -101,7 +102,7 @@ class SiteActivityAPI(API):
101
102
  # Always return a result even not complete
102
103
  # But log the error (ie. visible in sentry, silent for user)
103
104
  # Can happen when someone manually delete an object in DB (ie. without proper purge)
104
- # - Filter out private items (except for sysadmin users)
105
+ # - Filter out items not visible to the current user
105
106
  safe_items = []
106
107
  for item in qs.queryset.items:
107
108
  try:
@@ -109,10 +110,8 @@ class SiteActivityAPI(API):
109
110
  except DoesNotExist as e:
110
111
  log.error(e, exc_info=True)
111
112
  else:
112
- if hasattr(item.related_to, "private") and (
113
- current_user.is_anonymous or not current_user.sysadmin
114
- ):
115
- if item.related_to.private:
113
+ if isinstance(item.related_to, Owned):
114
+ if not OwnableReadPermission(item.related_to).can():
116
115
  continue
117
116
  safe_items.append(item)
118
117
  qs.queryset.items = safe_items
@@ -1,5 +1,3 @@
1
- import udata.core.dataservices.tasks # noqa
2
- import udata.core.dataset.tasks # noqa
3
1
  from udata.core.badges.tasks import update_badges
4
2
  from udata.core.constants import HVD
5
3
  from udata.core.dataservices.factories import DataserviceFactory
udata/core/csv.py CHANGED
@@ -5,6 +5,7 @@ from datetime import date, datetime
5
5
  from io import StringIO
6
6
 
7
7
  from flask import Response, stream_with_context
8
+ from mongoengine.queryset import QuerySet
8
9
 
9
10
  from udata.mongo import db
10
11
  from udata.utils import recursive_get
@@ -35,6 +36,10 @@ class Adapter(object):
35
36
  fields = None
36
37
 
37
38
  def __init__(self, queryset):
39
+ # no_cache() to avoid eating up too much RAM when iterating over large querysets.
40
+ # Applied here rather than upstream to preserve custom QuerySet methods (like with_badge).
41
+ if isinstance(queryset, QuerySet):
42
+ queryset = queryset.no_cache()
38
43
  self.queryset = queryset
39
44
  self._fields = None
40
45
 
@@ -12,6 +12,7 @@ from udata.auth import admin_permission
12
12
  from udata.core.access_type.constants import AccessType
13
13
  from udata.core.dataset.models import Dataset
14
14
  from udata.core.followers.api import FollowAPI
15
+ from udata.core.legal.mails import add_send_legal_notice_argument, send_legal_notice_on_deletion
15
16
  from udata.frontend.markdown import md
16
17
  from udata.i18n import gettext as _
17
18
  from udata.rdf import RDF_EXTENSIONS, graph_response, negociate_content
@@ -88,6 +89,9 @@ class DataservicesAtomFeedAPI(API):
88
89
  return response
89
90
 
90
91
 
92
+ dataservice_delete_parser = add_send_legal_notice_argument(api.parser())
93
+
94
+
91
95
  @ns.route("/<dataservice:dataservice>/", endpoint="dataservice")
92
96
  class DataserviceAPI(API):
93
97
  @api.doc("get_dataservice")
@@ -123,16 +127,19 @@ class DataserviceAPI(API):
123
127
 
124
128
  @api.secure
125
129
  @api.doc("delete_dataservice")
130
+ @api.expect(dataservice_delete_parser)
126
131
  @api.response(204, "dataservice deleted")
127
132
  def delete(self, dataservice):
133
+ args = dataservice_delete_parser.parse_args()
128
134
  if dataservice.deleted_at:
129
135
  api.abort(410, "dataservice has been deleted")
130
136
 
131
137
  dataservice.permissions["delete"].test()
138
+ send_legal_notice_on_deletion(dataservice, args)
139
+
132
140
  dataservice.deleted_at = datetime.utcnow()
133
141
  dataservice.metadata_modified_at = datetime.utcnow()
134
142
  dataservice.save()
135
-
136
143
  return "", 204
137
144
 
138
145
 
@@ -1,10 +1,7 @@
1
- from flask import request
2
-
3
1
  from udata import search
4
2
  from udata.api import API, apiv2
5
3
  from udata.core.access_type.models import AccessAudience
6
4
  from udata.core.dataservices.models import Dataservice, HarvestMetadata
7
- from udata.utils import multi_to_dict
8
5
 
9
6
  from .models import dataservice_permissions_fields
10
7
  from .search import DataserviceSearch
@@ -18,7 +15,7 @@ apiv2.inherit("AccessAudience (read)", AccessAudience.__read_fields__)
18
15
 
19
16
  ns = apiv2.namespace("dataservices", "Dataservice related operations")
20
17
 
21
- search_parser = DataserviceSearch.as_request_parser()
18
+ search_parser = DataserviceSearch.as_request_parser(store_missing=False)
22
19
 
23
20
 
24
21
  @ns.route("/search/", endpoint="dataservice_search")
@@ -30,5 +27,5 @@ class DataserviceSearchAPI(API):
30
27
  @apiv2.marshal_with(Dataservice.__page_fields__)
31
28
  def get(self):
32
29
  """Search all dataservices"""
33
- search_parser.parse_args()
34
- return search.query(DataserviceSearch, **multi_to_dict(request.args))
30
+ args = search_parser.parse_args()
31
+ return search.query(DataserviceSearch, **args)
@@ -201,7 +201,10 @@ class Dataservice(
201
201
  ),
202
202
  readonly=True,
203
203
  )
204
- description = field(db.StringField(default=""), description="In markdown")
204
+ description = field(
205
+ db.StringField(default=""),
206
+ markdown=True,
207
+ )
205
208
  base_api_url = field(db.URLField(), sortable=True)
206
209
 
207
210
  machine_documentation_url = field(
@@ -309,7 +312,7 @@ class Dataservice(
309
312
 
310
313
  @field(description="Link to the udata web page for this dataservice", show_as_ref=True)
311
314
  def self_web_url(self, **kwargs):
312
- return cdata_url(f"/dataservices/{self._link_id(**kwargs)}/", **kwargs)
315
+ return cdata_url(f"/dataservices/{self._link_id(**kwargs)}", **kwargs)
313
316
 
314
317
  __metrics_keys__ = [
315
318
  "discussions",
@@ -31,6 +31,7 @@ def dataservice_from_rdf(
31
31
  node,
32
32
  all_datasets: list[Dataset],
33
33
  remote_url_prefix: str | None = None,
34
+ dryrun: bool = False,
34
35
  ) -> Dataservice:
35
36
  """
36
37
  Create or update a dataservice from a RDF/DCAT graph
@@ -51,7 +52,7 @@ def dataservice_from_rdf(
51
52
  dataservice.machine_documentation_url = url_from_rdf(d, DCAT.endpointDescription)
52
53
 
53
54
  roles = [ # Imbricated list of contact points for each role
54
- contact_points_from_rdf(d, rdf_entity, role, dataservice)
55
+ contact_points_from_rdf(d, rdf_entity, role, dataservice, dryrun=dryrun)
55
56
  for rdf_entity, role in CONTACT_POINT_ENTITY_TO_ROLE.items()
56
57
  ]
57
58
  dataservice.contact_points = [ # Flattened list of contact points