udata 14.0.3.dev1__py3-none-any.whl → 14.7.3.dev4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- udata/api/__init__.py +2 -0
- udata/api_fields.py +120 -19
- udata/app.py +18 -20
- udata/auth/__init__.py +4 -7
- udata/auth/forms.py +3 -3
- udata/auth/views.py +13 -6
- udata/commands/dcat.py +1 -1
- udata/commands/serve.py +3 -11
- udata/core/activity/api.py +5 -6
- udata/core/badges/tests/test_tasks.py +0 -2
- udata/core/csv.py +5 -0
- udata/core/dataservices/api.py +8 -1
- udata/core/dataservices/apiv2.py +3 -6
- udata/core/dataservices/models.py +5 -2
- udata/core/dataservices/rdf.py +2 -1
- udata/core/dataservices/tasks.py +6 -2
- udata/core/dataset/api.py +30 -4
- udata/core/dataset/api_fields.py +1 -1
- udata/core/dataset/apiv2.py +1 -1
- udata/core/dataset/constants.py +2 -9
- udata/core/dataset/models.py +21 -9
- udata/core/dataset/permissions.py +31 -0
- udata/core/dataset/rdf.py +18 -16
- udata/core/dataset/tasks.py +16 -7
- udata/core/discussions/api.py +15 -1
- udata/core/discussions/models.py +6 -0
- udata/core/legal/__init__.py +0 -0
- udata/core/legal/mails.py +128 -0
- udata/core/organization/api.py +16 -5
- udata/core/organization/api_fields.py +3 -3
- udata/core/organization/apiv2.py +3 -4
- udata/core/organization/mails.py +1 -1
- udata/core/organization/models.py +40 -7
- udata/core/organization/notifications.py +84 -0
- udata/core/organization/permissions.py +1 -1
- udata/core/organization/tasks.py +3 -0
- udata/core/pages/models.py +49 -0
- udata/core/pages/tests/test_api.py +165 -1
- udata/core/post/api.py +25 -70
- udata/core/post/constants.py +8 -0
- udata/core/post/models.py +109 -17
- udata/core/post/tests/test_api.py +140 -3
- udata/core/post/tests/test_models.py +24 -0
- udata/core/reports/api.py +18 -0
- udata/core/reports/models.py +42 -2
- udata/core/reuse/api.py +8 -0
- udata/core/reuse/apiv2.py +3 -6
- udata/core/reuse/models.py +1 -1
- udata/core/spatial/forms.py +2 -2
- udata/core/topic/models.py +8 -2
- udata/core/user/api.py +10 -3
- udata/core/user/api_fields.py +3 -3
- udata/core/user/models.py +33 -8
- udata/features/notifications/api.py +7 -18
- udata/features/notifications/models.py +59 -0
- udata/features/notifications/tasks.py +25 -0
- udata/features/transfer/actions.py +2 -0
- udata/features/transfer/models.py +17 -0
- udata/features/transfer/notifications.py +96 -0
- udata/flask_mongoengine/engine.py +0 -4
- udata/flask_mongoengine/pagination.py +1 -1
- udata/frontend/markdown.py +2 -1
- udata/harvest/actions.py +20 -0
- udata/harvest/api.py +24 -7
- udata/harvest/backends/base.py +27 -1
- udata/harvest/backends/ckan/harvesters.py +21 -4
- udata/harvest/backends/dcat.py +4 -1
- udata/harvest/commands.py +33 -0
- udata/harvest/filters.py +17 -6
- udata/harvest/models.py +16 -0
- udata/harvest/permissions.py +27 -0
- udata/harvest/tests/ckan/test_ckan_backend.py +33 -0
- udata/harvest/tests/test_actions.py +46 -2
- udata/harvest/tests/test_api.py +161 -6
- udata/harvest/tests/test_base_backend.py +86 -1
- udata/harvest/tests/test_dcat_backend.py +68 -3
- udata/harvest/tests/test_filters.py +6 -0
- udata/i18n.py +1 -4
- udata/mail.py +14 -0
- udata/migrations/2021-08-17-harvest-integrity.py +23 -16
- udata/migrations/2025-10-31-create-membership-request-notifications.py +55 -0
- udata/migrations/2025-12-04-add-uuid-to-discussion-messages.py +28 -0
- udata/migrations/2025-12-16-create-transfer-request-notifications.py +69 -0
- udata/migrations/2026-01-14-add-default-kind-to-posts.py +17 -0
- udata/mongo/slug_fields.py +1 -1
- udata/rdf.py +65 -11
- udata/routing.py +2 -2
- udata/settings.py +11 -0
- udata/tasks.py +2 -0
- udata/templates/mail/message.html +3 -1
- udata/tests/api/__init__.py +7 -17
- udata/tests/api/test_activities_api.py +36 -0
- udata/tests/api/test_datasets_api.py +69 -0
- udata/tests/api/test_organizations_api.py +0 -3
- udata/tests/api/test_reports_api.py +157 -0
- udata/tests/api/test_user_api.py +1 -1
- udata/tests/apiv2/test_dataservices.py +14 -0
- udata/tests/apiv2/test_organizations.py +9 -0
- udata/tests/apiv2/test_reuses.py +11 -0
- udata/tests/cli/test_cli_base.py +0 -1
- udata/tests/dataservice/test_dataservice_tasks.py +29 -0
- udata/tests/dataset/test_dataset_model.py +13 -1
- udata/tests/dataset/test_dataset_rdf.py +164 -5
- udata/tests/dataset/test_dataset_tasks.py +25 -0
- udata/tests/frontend/test_auth.py +58 -1
- udata/tests/frontend/test_csv.py +0 -3
- udata/tests/helpers.py +31 -27
- udata/tests/organization/test_notifications.py +67 -2
- udata/tests/search/test_search_integration.py +70 -0
- udata/tests/site/test_site_csv_exports.py +22 -10
- udata/tests/test_activity.py +9 -9
- udata/tests/test_api_fields.py +10 -0
- udata/tests/test_discussions.py +5 -5
- udata/tests/test_legal_mails.py +359 -0
- udata/tests/test_notifications.py +15 -57
- udata/tests/test_notifications_task.py +43 -0
- udata/tests/test_owned.py +81 -1
- udata/tests/test_transfer.py +181 -2
- udata/tests/test_uris.py +33 -0
- udata/translations/ar/LC_MESSAGES/udata.mo +0 -0
- udata/translations/ar/LC_MESSAGES/udata.po +309 -158
- udata/translations/de/LC_MESSAGES/udata.mo +0 -0
- udata/translations/de/LC_MESSAGES/udata.po +313 -160
- udata/translations/es/LC_MESSAGES/udata.mo +0 -0
- udata/translations/es/LC_MESSAGES/udata.po +312 -160
- udata/translations/fr/LC_MESSAGES/udata.mo +0 -0
- udata/translations/fr/LC_MESSAGES/udata.po +475 -202
- udata/translations/it/LC_MESSAGES/udata.mo +0 -0
- udata/translations/it/LC_MESSAGES/udata.po +317 -162
- udata/translations/pt/LC_MESSAGES/udata.mo +0 -0
- udata/translations/pt/LC_MESSAGES/udata.po +315 -161
- udata/translations/sr/LC_MESSAGES/udata.mo +0 -0
- udata/translations/sr/LC_MESSAGES/udata.po +323 -164
- udata/translations/udata.pot +169 -124
- udata/uris.py +0 -2
- udata/utils.py +23 -0
- udata-14.7.3.dev4.dist-info/METADATA +109 -0
- {udata-14.0.3.dev1.dist-info → udata-14.7.3.dev4.dist-info}/RECORD +142 -135
- udata/core/post/forms.py +0 -30
- udata/flask_mongoengine/json.py +0 -38
- udata/templates/mail/base.html +0 -105
- udata/templates/mail/base.txt +0 -6
- udata/templates/mail/button.html +0 -3
- udata/templates/mail/layouts/1-column.html +0 -19
- udata/templates/mail/layouts/2-columns.html +0 -20
- udata/templates/mail/layouts/center-panel.html +0 -16
- udata-14.0.3.dev1.dist-info/METADATA +0 -132
- {udata-14.0.3.dev1.dist-info → udata-14.7.3.dev4.dist-info}/WHEEL +0 -0
- {udata-14.0.3.dev1.dist-info → udata-14.7.3.dev4.dist-info}/entry_points.txt +0 -0
- {udata-14.0.3.dev1.dist-info → udata-14.7.3.dev4.dist-info}/licenses/LICENSE +0 -0
- {udata-14.0.3.dev1.dist-info → udata-14.7.3.dev4.dist-info}/top_level.txt +0 -0
udata/api/__init__.py
CHANGED
|
@@ -121,6 +121,8 @@ class UDataApi(Api):
|
|
|
121
121
|
if "application/json" not in request.headers.get("Content-Type", ""):
|
|
122
122
|
errors = {"Content-Type": "expecting application/json"}
|
|
123
123
|
self.abort(400, errors=errors)
|
|
124
|
+
if not isinstance(request.json, dict):
|
|
125
|
+
self.abort(400, errors={"request": "expecting a JSON object"})
|
|
124
126
|
form = form_cls.from_json(request.json, obj=obj, instance=obj, meta={"csrf": False})
|
|
125
127
|
if not form.validate():
|
|
126
128
|
self.abort(400, errors=form.errors)
|
udata/api_fields.py
CHANGED
|
@@ -29,7 +29,7 @@ import flask_restx.fields as restx_fields
|
|
|
29
29
|
import mongoengine
|
|
30
30
|
import mongoengine.fields as mongo_fields
|
|
31
31
|
from bson import DBRef, ObjectId
|
|
32
|
-
from flask import Request
|
|
32
|
+
from flask import Request, request
|
|
33
33
|
from flask_restx import marshal
|
|
34
34
|
from flask_restx.inputs import boolean
|
|
35
35
|
from flask_restx.reqparse import RequestParser
|
|
@@ -40,6 +40,32 @@ from udata.api import api, base_reference
|
|
|
40
40
|
from udata.mongo.errors import FieldValidationError
|
|
41
41
|
from udata.mongo.queryset import DBPaginator, UDataQuerySet
|
|
42
42
|
|
|
43
|
+
|
|
44
|
+
def required_if(**conditions):
|
|
45
|
+
"""Check helper that makes a field required when other fields have specific values.
|
|
46
|
+
|
|
47
|
+
Usage:
|
|
48
|
+
page_id = field(
|
|
49
|
+
db.ReferenceField("Page"),
|
|
50
|
+
checks=[required_if(body_type="blocs")],
|
|
51
|
+
)
|
|
52
|
+
"""
|
|
53
|
+
|
|
54
|
+
def check(value, data, field, obj, **_kwargs):
|
|
55
|
+
if value is not None:
|
|
56
|
+
return
|
|
57
|
+
for condition_field, condition_value in conditions.items():
|
|
58
|
+
actual_condition = data.get(condition_field, getattr(obj, condition_field, None))
|
|
59
|
+
if actual_condition == condition_value:
|
|
60
|
+
raise FieldValidationError(
|
|
61
|
+
f"'{field}' is required when '{condition_field}' is '{condition_value}'",
|
|
62
|
+
field=field,
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
check.run_even_if_missing = True
|
|
66
|
+
return check
|
|
67
|
+
|
|
68
|
+
|
|
43
69
|
lazy_reference = api.model(
|
|
44
70
|
"LazyReference",
|
|
45
71
|
{
|
|
@@ -55,8 +81,8 @@ classes_by_parents = {}
|
|
|
55
81
|
|
|
56
82
|
|
|
57
83
|
class GenericField(restx_fields.Raw):
|
|
58
|
-
def __init__(self, fields_by_type):
|
|
59
|
-
super().__init__(
|
|
84
|
+
def __init__(self, fields_by_type, **kwargs):
|
|
85
|
+
super(GenericField, self).__init__(**kwargs)
|
|
60
86
|
self.default = None
|
|
61
87
|
self.fields_by_type = fields_by_type
|
|
62
88
|
|
|
@@ -79,13 +105,15 @@ def convert_db_to_field(key, field, info) -> tuple[Callable | None, Callable | N
|
|
|
79
105
|
user-supplied overrides, setting the readonly flag…), it's easier to have to do this only once at the end of the function.
|
|
80
106
|
|
|
81
107
|
"""
|
|
108
|
+
from udata.mongo.engine import db
|
|
109
|
+
|
|
82
110
|
params: dict = {}
|
|
83
111
|
params["required"] = field.required
|
|
84
112
|
|
|
85
113
|
read_params: dict = {}
|
|
86
114
|
write_params: dict = {}
|
|
87
115
|
|
|
88
|
-
constructor: Callable
|
|
116
|
+
constructor: Callable | None = None
|
|
89
117
|
constructor_read: Callable | None = None
|
|
90
118
|
constructor_write: Callable | None = None
|
|
91
119
|
|
|
@@ -204,13 +232,34 @@ def convert_db_to_field(key, field, info) -> tuple[Callable | None, Callable | N
|
|
|
204
232
|
def constructor_write(**kwargs):
|
|
205
233
|
return restx_fields.List(field_write, **kwargs)
|
|
206
234
|
|
|
207
|
-
elif isinstance(
|
|
208
|
-
field, (mongo_fields.GenericReferenceField, mongoengine.fields.GenericLazyReferenceField)
|
|
209
|
-
):
|
|
235
|
+
elif isinstance(field, mongoengine.fields.GenericLazyReferenceField):
|
|
210
236
|
|
|
211
237
|
def constructor(**kwargs):
|
|
212
238
|
return restx_fields.Nested(lazy_reference, **kwargs)
|
|
213
239
|
|
|
240
|
+
elif isinstance(field, mongo_fields.GenericReferenceField):
|
|
241
|
+
if field.choices:
|
|
242
|
+
generic_fields = {}
|
|
243
|
+
for cls in field.choices:
|
|
244
|
+
cls = db.resolve_model(cls) if isinstance(cls, str) else cls
|
|
245
|
+
generic_fields[cls.__name__] = convert_db_to_field(
|
|
246
|
+
f"{key}.{cls.__name__}",
|
|
247
|
+
# Instead of having GenericReferenceField() we'll create fields for each
|
|
248
|
+
# of the subclasses with ReferenceField(Organization)…
|
|
249
|
+
mongoengine.fields.ReferenceField(cls),
|
|
250
|
+
info,
|
|
251
|
+
)
|
|
252
|
+
|
|
253
|
+
def constructor_read(**kwargs):
|
|
254
|
+
return GenericField({k: v[0].model for k, v in generic_fields.items()}, **kwargs)
|
|
255
|
+
|
|
256
|
+
def constructor_write(**kwargs):
|
|
257
|
+
return GenericField({k: v[1].model for k, v in generic_fields.items()}, **kwargs)
|
|
258
|
+
else:
|
|
259
|
+
|
|
260
|
+
def constructor(**kwargs):
|
|
261
|
+
return restx_fields.Nested(lazy_reference, **kwargs)
|
|
262
|
+
|
|
214
263
|
elif isinstance(field, mongo_fields.ReferenceField | mongo_fields.LazyReferenceField):
|
|
215
264
|
# For reference we accept while writing a String representing the ID of the referenced model.
|
|
216
265
|
# For reading, if the user supplied a `nested_fields` (RestX model), we use it to convert
|
|
@@ -230,6 +279,23 @@ def convert_db_to_field(key, field, info) -> tuple[Callable | None, Callable | N
|
|
|
230
279
|
|
|
231
280
|
write_params["description"] = "ID of the reference"
|
|
232
281
|
constructor_write = restx_fields.String
|
|
282
|
+
elif isinstance(field, mongo_fields.GenericEmbeddedDocumentField):
|
|
283
|
+
generic_fields = {
|
|
284
|
+
cls.__name__: convert_db_to_field(
|
|
285
|
+
f"{key}.{cls.__name__}",
|
|
286
|
+
# Instead of having GenericEmbeddedDocumentField() we'll create fields for each
|
|
287
|
+
# of the subclasses with EmbededdDocumentField(MembershipRequestNotificationDetails)…
|
|
288
|
+
mongoengine.fields.EmbeddedDocumentField(cls),
|
|
289
|
+
info,
|
|
290
|
+
)
|
|
291
|
+
for cls in field.choices
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
def constructor_read(**kwargs):
|
|
295
|
+
return GenericField({k: v[0].model for k, v in generic_fields.items()}, **kwargs)
|
|
296
|
+
|
|
297
|
+
def constructor_write(**kwargs):
|
|
298
|
+
return GenericField({k: v[1].model for k, v in generic_fields.items()}, **kwargs)
|
|
233
299
|
elif isinstance(field, mongo_fields.EmbeddedDocumentField):
|
|
234
300
|
nested_fields = info.get("nested_fields")
|
|
235
301
|
if nested_fields is not None:
|
|
@@ -322,12 +388,12 @@ def generate_fields(**kwargs) -> Callable:
|
|
|
322
388
|
write_fields: dict = {}
|
|
323
389
|
ref_fields: dict = {}
|
|
324
390
|
sortables: list = kwargs.get("additional_sorts", [])
|
|
391
|
+
default_sort: list = kwargs.get("default_sort", None)
|
|
325
392
|
|
|
326
393
|
filterables: list[dict] = kwargs.get("standalone_filters", [])
|
|
327
394
|
nested_filters: dict[str, dict] = get_fields_with_nested_filters(
|
|
328
395
|
kwargs.get("nested_filters", {})
|
|
329
396
|
)
|
|
330
|
-
|
|
331
397
|
if issubclass(cls, db.Document) or issubclass(cls, db.DynamicDocument):
|
|
332
398
|
read_fields["id"] = restx_fields.String(required=True, readonly=True)
|
|
333
399
|
|
|
@@ -472,6 +538,7 @@ def generate_fields(**kwargs) -> Callable:
|
|
|
472
538
|
type=str,
|
|
473
539
|
location="args",
|
|
474
540
|
choices=choices,
|
|
541
|
+
default=default_sort,
|
|
475
542
|
help="The field (and direction) on which sorting apply",
|
|
476
543
|
)
|
|
477
544
|
|
|
@@ -511,6 +578,9 @@ def generate_fields(**kwargs) -> Callable:
|
|
|
511
578
|
phrase_query: str = " ".join([f'"{elem}"' for elem in args["q"].split(" ")])
|
|
512
579
|
base_query = base_query.search_text(phrase_query)
|
|
513
580
|
|
|
581
|
+
if "sort" not in request.args:
|
|
582
|
+
base_query = base_query.order_by("$text_score")
|
|
583
|
+
|
|
514
584
|
for filterable in filterables:
|
|
515
585
|
# If it's from an `nested_filter`, use the custom label instead of the key,
|
|
516
586
|
# eg use `organization_badge` instead of `organization.badges` which is
|
|
@@ -637,6 +707,19 @@ def field(
|
|
|
637
707
|
return inner
|
|
638
708
|
|
|
639
709
|
|
|
710
|
+
def run_check(check, value, key, obj, data):
|
|
711
|
+
check(
|
|
712
|
+
value,
|
|
713
|
+
**{
|
|
714
|
+
"is_creation": obj._created,
|
|
715
|
+
"is_update": not obj._created,
|
|
716
|
+
"field": key,
|
|
717
|
+
"obj": obj,
|
|
718
|
+
"data": data,
|
|
719
|
+
},
|
|
720
|
+
)
|
|
721
|
+
|
|
722
|
+
|
|
640
723
|
def patch(obj, request) -> type:
|
|
641
724
|
"""Patch the object with the data from the request.
|
|
642
725
|
|
|
@@ -646,12 +729,16 @@ def patch(obj, request) -> type:
|
|
|
646
729
|
from udata.mongo.engine import db
|
|
647
730
|
|
|
648
731
|
data = request.json if isinstance(request, Request) else request
|
|
732
|
+
|
|
649
733
|
for key, value in data.items():
|
|
650
734
|
field = obj.__write_fields__.get(key)
|
|
651
735
|
if field is not None and not field.readonly:
|
|
652
736
|
model_attribute = getattr(obj.__class__, key)
|
|
653
737
|
info = getattr(model_attribute, "__additional_field_info__", {})
|
|
654
738
|
|
|
739
|
+
if value == "" and isinstance(model_attribute, mongo_fields.StringField):
|
|
740
|
+
value = None
|
|
741
|
+
|
|
655
742
|
if hasattr(model_attribute, "from_input"):
|
|
656
743
|
value = model_attribute.from_input(value)
|
|
657
744
|
elif isinstance(model_attribute, mongoengine.fields.ListField) and isinstance(
|
|
@@ -677,6 +764,13 @@ def patch(obj, request) -> type:
|
|
|
677
764
|
value["id"],
|
|
678
765
|
document_type=db.resolve_model(value["class"]),
|
|
679
766
|
)
|
|
767
|
+
elif value and isinstance(
|
|
768
|
+
model_attribute,
|
|
769
|
+
mongoengine.fields.GenericEmbeddedDocumentField,
|
|
770
|
+
):
|
|
771
|
+
generic_key = info.get("generic_key", DEFAULT_GENERIC_KEY)
|
|
772
|
+
embedded_field = classes_by_names[value[generic_key]]
|
|
773
|
+
value = patch(embedded_field(), value)
|
|
680
774
|
elif value and isinstance(
|
|
681
775
|
model_attribute,
|
|
682
776
|
mongoengine.fields.EmbeddedDocumentField,
|
|
@@ -703,23 +797,30 @@ def patch(obj, request) -> type:
|
|
|
703
797
|
|
|
704
798
|
value = objects
|
|
705
799
|
|
|
706
|
-
#
|
|
707
|
-
#
|
|
800
|
+
# Run checks if value is modified.
|
|
801
|
+
# We run checks here (before setattr) to compare old vs new value.
|
|
708
802
|
checks = info.get("checks", [])
|
|
709
|
-
|
|
710
803
|
if is_value_modified(getattr(obj, key), value):
|
|
711
804
|
for check in checks:
|
|
712
|
-
check
|
|
713
|
-
value,
|
|
714
|
-
**{
|
|
715
|
-
"is_creation": obj._created,
|
|
716
|
-
"is_update": not obj._created,
|
|
717
|
-
"field": key,
|
|
718
|
-
},
|
|
719
|
-
) # TODO add other model attributes in function parameters
|
|
805
|
+
run_check(check, value, key, obj, data)
|
|
720
806
|
|
|
721
807
|
setattr(obj, key, value)
|
|
722
808
|
|
|
809
|
+
# Run checks marked with `run_even_if_missing` on fields not in request.
|
|
810
|
+
# Some checks (like `required_if`) need to run even when their field is absent
|
|
811
|
+
# from the request, because they validate cross-field constraints based on
|
|
812
|
+
# other fields in the request (e.g. "page_id is required if body_type is blocs").
|
|
813
|
+
for key, _, info in get_fields(obj.__class__):
|
|
814
|
+
if key in data:
|
|
815
|
+
continue
|
|
816
|
+
checks = info.get("checks", [])
|
|
817
|
+
value = getattr(obj, key, None)
|
|
818
|
+
|
|
819
|
+
for check in checks:
|
|
820
|
+
if not getattr(check, "run_even_if_missing", False):
|
|
821
|
+
continue
|
|
822
|
+
run_check(check, value, key, obj, data)
|
|
823
|
+
|
|
723
824
|
return obj
|
|
724
825
|
|
|
725
826
|
|
udata/app.py
CHANGED
|
@@ -1,26 +1,28 @@
|
|
|
1
|
-
import datetime
|
|
2
1
|
import logging
|
|
3
2
|
import os
|
|
4
3
|
import types
|
|
4
|
+
from datetime import datetime
|
|
5
5
|
from importlib.metadata import entry_points
|
|
6
6
|
from os.path import abspath, dirname, exists, isfile, join
|
|
7
7
|
|
|
8
8
|
import bson
|
|
9
|
+
from bson import json_util
|
|
9
10
|
from flask import Blueprint as BaseBlueprint
|
|
10
11
|
from flask import (
|
|
11
12
|
Flask,
|
|
12
13
|
abort,
|
|
13
14
|
g,
|
|
14
|
-
json,
|
|
15
15
|
jsonify,
|
|
16
16
|
make_response,
|
|
17
17
|
render_template,
|
|
18
18
|
request,
|
|
19
19
|
send_from_directory,
|
|
20
20
|
)
|
|
21
|
+
from flask.json.provider import DefaultJSONProvider
|
|
21
22
|
from flask_caching import Cache
|
|
22
23
|
from flask_wtf.csrf import CSRFProtect
|
|
23
|
-
from
|
|
24
|
+
from mongoengine import EmbeddedDocument
|
|
25
|
+
from mongoengine.base import BaseDocument
|
|
24
26
|
from werkzeug.exceptions import NotFound
|
|
25
27
|
from werkzeug.middleware.proxy_fix import ProxyFix
|
|
26
28
|
|
|
@@ -108,9 +110,9 @@ class Blueprint(BaseBlueprint):
|
|
|
108
110
|
return wrapper
|
|
109
111
|
|
|
110
112
|
|
|
111
|
-
class
|
|
113
|
+
class UdataJsonProvider(DefaultJSONProvider):
|
|
112
114
|
"""
|
|
113
|
-
A
|
|
115
|
+
A JSONProvider subclass to encode unsupported types:
|
|
114
116
|
|
|
115
117
|
- ObjectId
|
|
116
118
|
- datetime
|
|
@@ -120,21 +122,16 @@ class UDataJsonEncoder(json.JSONEncoder):
|
|
|
120
122
|
Ensure an app context is always present.
|
|
121
123
|
"""
|
|
122
124
|
|
|
123
|
-
|
|
124
|
-
|
|
125
|
+
@staticmethod
|
|
126
|
+
def default(obj):
|
|
127
|
+
if isinstance(obj, BaseDocument) or isinstance(obj, EmbeddedDocument):
|
|
128
|
+
return json_util._json_convert(obj.to_mongo())
|
|
129
|
+
elif isinstance(obj, bson.ObjectId):
|
|
125
130
|
return str(obj)
|
|
126
|
-
elif isinstance(obj,
|
|
127
|
-
return str(obj)
|
|
128
|
-
elif isinstance(obj, datetime.datetime):
|
|
131
|
+
elif isinstance(obj, datetime):
|
|
129
132
|
return obj.isoformat()
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
elif hasattr(obj, "serialize"):
|
|
133
|
-
return obj.serialize()
|
|
134
|
-
# Serialize Raw data for Document and EmbeddedDocument.
|
|
135
|
-
elif hasattr(obj, "_data"):
|
|
136
|
-
return obj._data
|
|
137
|
-
return super(UDataJsonEncoder, self).default(obj)
|
|
133
|
+
|
|
134
|
+
return super(UdataJsonProvider, UdataJsonProvider).default(obj)
|
|
138
135
|
|
|
139
136
|
|
|
140
137
|
# These loggers are very verbose
|
|
@@ -166,10 +163,11 @@ def create_app(config="udata.settings.Defaults", override=None, init_logging=ini
|
|
|
166
163
|
if override:
|
|
167
164
|
app.config.from_object(override)
|
|
168
165
|
|
|
169
|
-
app.
|
|
166
|
+
app.json_provider_class = UdataJsonProvider
|
|
167
|
+
app.json = app.json_provider_class(app)
|
|
170
168
|
|
|
171
169
|
# `ujson` doesn't support `cls` parameter https://github.com/ultrajson/ultrajson/issues/124
|
|
172
|
-
app.config["RESTX_JSON"] = {"
|
|
170
|
+
app.config["RESTX_JSON"] = {"default": UdataJsonProvider.default}
|
|
173
171
|
|
|
174
172
|
app.debug = app.config["DEBUG"] and not app.config["TESTING"]
|
|
175
173
|
|
udata/auth/__init__.py
CHANGED
|
@@ -26,9 +26,6 @@ def render_security_template(template_name_or_list, **kwargs):
|
|
|
26
26
|
return render_template(template_name_or_list, **kwargs)
|
|
27
27
|
|
|
28
28
|
|
|
29
|
-
security = Security()
|
|
30
|
-
|
|
31
|
-
|
|
32
29
|
class Permission(BasePermission):
|
|
33
30
|
def __init__(self, *needs):
|
|
34
31
|
"""Let administrator bypass all permissions"""
|
|
@@ -84,15 +81,13 @@ def init_app(app):
|
|
|
84
81
|
# Same logic as in our own mail system :DisableMail
|
|
85
82
|
debug = app.config.get("DEBUG", False)
|
|
86
83
|
send_mail = app.config.get("SEND_MAIL", not debug)
|
|
87
|
-
mail_util_cls =
|
|
84
|
+
mail_util_cls = mail_util.MailUtil if send_mail else NoopMailUtil
|
|
88
85
|
|
|
89
|
-
security
|
|
90
|
-
app,
|
|
86
|
+
security = Security(
|
|
91
87
|
datastore,
|
|
92
88
|
register_blueprint=False,
|
|
93
89
|
render_template=render_security_template,
|
|
94
90
|
login_form=ExtendedLoginForm,
|
|
95
|
-
confirm_register_form=ExtendedRegisterForm,
|
|
96
91
|
register_form=ExtendedRegisterForm,
|
|
97
92
|
reset_password_form=ExtendedResetPasswordForm,
|
|
98
93
|
forgot_password_form=ExtendedForgotPasswordForm,
|
|
@@ -100,6 +95,8 @@ def init_app(app):
|
|
|
100
95
|
mail_util_cls=mail_util_cls,
|
|
101
96
|
)
|
|
102
97
|
|
|
98
|
+
security.init_app(app, datastore, register_blueprint=False)
|
|
99
|
+
|
|
103
100
|
security_bp = create_security_blueprint(app, app.extensions["security"], "security_blueprint")
|
|
104
101
|
|
|
105
102
|
app.register_blueprint(security_bp)
|
udata/auth/forms.py
CHANGED
|
@@ -8,7 +8,7 @@ from flask_security.forms import (
|
|
|
8
8
|
ForgotPasswordForm,
|
|
9
9
|
Form,
|
|
10
10
|
LoginForm,
|
|
11
|
-
|
|
11
|
+
RegisterFormV2,
|
|
12
12
|
ResetPasswordForm,
|
|
13
13
|
)
|
|
14
14
|
|
|
@@ -31,7 +31,7 @@ class WithCaptcha:
|
|
|
31
31
|
return False
|
|
32
32
|
|
|
33
33
|
|
|
34
|
-
class ExtendedRegisterForm(WithCaptcha,
|
|
34
|
+
class ExtendedRegisterForm(WithCaptcha, RegisterFormV2):
|
|
35
35
|
first_name = fields.StringField(
|
|
36
36
|
_("First name"),
|
|
37
37
|
[
|
|
@@ -49,7 +49,7 @@ class ExtendedRegisterForm(WithCaptcha, RegisterForm):
|
|
|
49
49
|
accept_conditions = fields.BooleanField(
|
|
50
50
|
_("J'accepte les conditions générales d'utilisation"),
|
|
51
51
|
validators=[
|
|
52
|
-
validators.DataRequired(message=_("
|
|
52
|
+
validators.DataRequired(message=_("You must accept the terms of use to continue."))
|
|
53
53
|
],
|
|
54
54
|
)
|
|
55
55
|
|
udata/auth/views.py
CHANGED
|
@@ -57,11 +57,15 @@ def confirm_change_email_token_status(token):
|
|
|
57
57
|
token, "confirm", get_within_delta("CONFIRM_EMAIL_WITHIN")
|
|
58
58
|
)
|
|
59
59
|
new_email = None
|
|
60
|
+
user = None
|
|
60
61
|
|
|
61
62
|
if not invalid and token_data:
|
|
62
|
-
|
|
63
|
-
user = _datastore.find_user(fs_uniquifier=
|
|
64
|
-
|
|
63
|
+
user_uniquifier, token_email_hash, new_email = token_data
|
|
64
|
+
user = _datastore.find_user(fs_uniquifier=user_uniquifier)
|
|
65
|
+
if user is None:
|
|
66
|
+
invalid = True
|
|
67
|
+
else:
|
|
68
|
+
invalid = not verify_hash(token_email_hash, user.email)
|
|
65
69
|
|
|
66
70
|
return expired, invalid, user, new_email
|
|
67
71
|
|
|
@@ -84,6 +88,11 @@ def confirm_change_email(token):
|
|
|
84
88
|
if flash:
|
|
85
89
|
return redirect(homepage_url(flash=flash, flash_data=flash_data))
|
|
86
90
|
|
|
91
|
+
# Check if the new email is already taken by another user
|
|
92
|
+
existing_user = _datastore.find_user(email=new_email)
|
|
93
|
+
if existing_user and existing_user.id != user.id:
|
|
94
|
+
return redirect(homepage_url(flash="change_email_already_taken"))
|
|
95
|
+
|
|
87
96
|
if user != current_user:
|
|
88
97
|
logout_user()
|
|
89
98
|
login_user(user)
|
|
@@ -140,10 +149,8 @@ def create_security_blueprint(app, state, import_name):
|
|
|
140
149
|
This creates an I18nBlueprint to use as a base.
|
|
141
150
|
"""
|
|
142
151
|
bp = I18nBlueprint(
|
|
143
|
-
|
|
152
|
+
"security",
|
|
144
153
|
import_name,
|
|
145
|
-
url_prefix=state.url_prefix,
|
|
146
|
-
subdomain=state.subdomain,
|
|
147
154
|
template_folder="templates",
|
|
148
155
|
)
|
|
149
156
|
|
udata/commands/dcat.py
CHANGED
|
@@ -85,7 +85,7 @@ def parse_url(url, csw, iso, quiet=False, rid=""):
|
|
|
85
85
|
echo("Item kwargs: {}".format(yellow(item.kwargs)))
|
|
86
86
|
node = backend.get_node_from_item(graph, item)
|
|
87
87
|
dataset = MockDatasetFactory()
|
|
88
|
-
dataset = dataset_from_rdf(graph, dataset, node=node)
|
|
88
|
+
dataset = dataset_from_rdf(graph, dataset, node=node, dryrun=True)
|
|
89
89
|
echo("")
|
|
90
90
|
echo(green("Dataset found!"))
|
|
91
91
|
echo("Title: {}".format(yellow(dataset)))
|
udata/commands/serve.py
CHANGED
|
@@ -3,7 +3,7 @@ import os
|
|
|
3
3
|
|
|
4
4
|
import click
|
|
5
5
|
from flask import current_app
|
|
6
|
-
from flask.cli import
|
|
6
|
+
from flask.cli import pass_script_info
|
|
7
7
|
from werkzeug.serving import run_simple
|
|
8
8
|
|
|
9
9
|
from udata.commands import cli
|
|
@@ -26,17 +26,11 @@ log = logging.getLogger(__name__)
|
|
|
26
26
|
default=None,
|
|
27
27
|
help="Enable or disable the debugger. By default the debugger is active if debug is enabled.",
|
|
28
28
|
)
|
|
29
|
-
@click.option(
|
|
30
|
-
"--eager-loading/--lazy-loader",
|
|
31
|
-
default=None,
|
|
32
|
-
help="Enable or disable eager loading. By default eager "
|
|
33
|
-
"loading is enabled if the reloader is disabled.",
|
|
34
|
-
)
|
|
35
29
|
@click.option(
|
|
36
30
|
"--with-threads/--without-threads", default=True, help="Enable or disable multithreading."
|
|
37
31
|
)
|
|
38
32
|
@pass_script_info
|
|
39
|
-
def serve(info, host, port, reload, debugger,
|
|
33
|
+
def serve(info, host, port, reload, debugger, with_threads):
|
|
40
34
|
"""
|
|
41
35
|
Runs a local udata development server.
|
|
42
36
|
|
|
@@ -62,10 +56,8 @@ def serve(info, host, port, reload, debugger, eager_loading, with_threads):
|
|
|
62
56
|
reload = bool(debug)
|
|
63
57
|
if debugger is None:
|
|
64
58
|
debugger = bool(debug)
|
|
65
|
-
if eager_loading is None:
|
|
66
|
-
eager_loading = not reload
|
|
67
59
|
|
|
68
|
-
app =
|
|
60
|
+
app = info.load_app()
|
|
69
61
|
|
|
70
62
|
settings = os.environ.get("UDATA_SETTINGS", os.path.join(os.getcwd(), "udata.cfg"))
|
|
71
63
|
extra_files = [settings]
|
udata/core/activity/api.py
CHANGED
|
@@ -4,8 +4,9 @@ from bson import ObjectId
|
|
|
4
4
|
from mongoengine.errors import DoesNotExist
|
|
5
5
|
|
|
6
6
|
from udata.api import API, api, fields
|
|
7
|
-
from udata.
|
|
7
|
+
from udata.core.dataset.permissions import OwnableReadPermission
|
|
8
8
|
from udata.core.organization.api_fields import org_ref_fields
|
|
9
|
+
from udata.core.owned import Owned
|
|
9
10
|
from udata.core.user.api_fields import user_ref_fields
|
|
10
11
|
from udata.models import Activity, db
|
|
11
12
|
|
|
@@ -101,7 +102,7 @@ class SiteActivityAPI(API):
|
|
|
101
102
|
# Always return a result even not complete
|
|
102
103
|
# But log the error (ie. visible in sentry, silent for user)
|
|
103
104
|
# Can happen when someone manually delete an object in DB (ie. without proper purge)
|
|
104
|
-
# - Filter out
|
|
105
|
+
# - Filter out items not visible to the current user
|
|
105
106
|
safe_items = []
|
|
106
107
|
for item in qs.queryset.items:
|
|
107
108
|
try:
|
|
@@ -109,10 +110,8 @@ class SiteActivityAPI(API):
|
|
|
109
110
|
except DoesNotExist as e:
|
|
110
111
|
log.error(e, exc_info=True)
|
|
111
112
|
else:
|
|
112
|
-
if
|
|
113
|
-
|
|
114
|
-
):
|
|
115
|
-
if item.related_to.private:
|
|
113
|
+
if isinstance(item.related_to, Owned):
|
|
114
|
+
if not OwnableReadPermission(item.related_to).can():
|
|
116
115
|
continue
|
|
117
116
|
safe_items.append(item)
|
|
118
117
|
qs.queryset.items = safe_items
|
udata/core/csv.py
CHANGED
|
@@ -5,6 +5,7 @@ from datetime import date, datetime
|
|
|
5
5
|
from io import StringIO
|
|
6
6
|
|
|
7
7
|
from flask import Response, stream_with_context
|
|
8
|
+
from mongoengine.queryset import QuerySet
|
|
8
9
|
|
|
9
10
|
from udata.mongo import db
|
|
10
11
|
from udata.utils import recursive_get
|
|
@@ -35,6 +36,10 @@ class Adapter(object):
|
|
|
35
36
|
fields = None
|
|
36
37
|
|
|
37
38
|
def __init__(self, queryset):
|
|
39
|
+
# no_cache() to avoid eating up too much RAM when iterating over large querysets.
|
|
40
|
+
# Applied here rather than upstream to preserve custom QuerySet methods (like with_badge).
|
|
41
|
+
if isinstance(queryset, QuerySet):
|
|
42
|
+
queryset = queryset.no_cache()
|
|
38
43
|
self.queryset = queryset
|
|
39
44
|
self._fields = None
|
|
40
45
|
|
udata/core/dataservices/api.py
CHANGED
|
@@ -12,6 +12,7 @@ from udata.auth import admin_permission
|
|
|
12
12
|
from udata.core.access_type.constants import AccessType
|
|
13
13
|
from udata.core.dataset.models import Dataset
|
|
14
14
|
from udata.core.followers.api import FollowAPI
|
|
15
|
+
from udata.core.legal.mails import add_send_legal_notice_argument, send_legal_notice_on_deletion
|
|
15
16
|
from udata.frontend.markdown import md
|
|
16
17
|
from udata.i18n import gettext as _
|
|
17
18
|
from udata.rdf import RDF_EXTENSIONS, graph_response, negociate_content
|
|
@@ -88,6 +89,9 @@ class DataservicesAtomFeedAPI(API):
|
|
|
88
89
|
return response
|
|
89
90
|
|
|
90
91
|
|
|
92
|
+
dataservice_delete_parser = add_send_legal_notice_argument(api.parser())
|
|
93
|
+
|
|
94
|
+
|
|
91
95
|
@ns.route("/<dataservice:dataservice>/", endpoint="dataservice")
|
|
92
96
|
class DataserviceAPI(API):
|
|
93
97
|
@api.doc("get_dataservice")
|
|
@@ -123,16 +127,19 @@ class DataserviceAPI(API):
|
|
|
123
127
|
|
|
124
128
|
@api.secure
|
|
125
129
|
@api.doc("delete_dataservice")
|
|
130
|
+
@api.expect(dataservice_delete_parser)
|
|
126
131
|
@api.response(204, "dataservice deleted")
|
|
127
132
|
def delete(self, dataservice):
|
|
133
|
+
args = dataservice_delete_parser.parse_args()
|
|
128
134
|
if dataservice.deleted_at:
|
|
129
135
|
api.abort(410, "dataservice has been deleted")
|
|
130
136
|
|
|
131
137
|
dataservice.permissions["delete"].test()
|
|
138
|
+
send_legal_notice_on_deletion(dataservice, args)
|
|
139
|
+
|
|
132
140
|
dataservice.deleted_at = datetime.utcnow()
|
|
133
141
|
dataservice.metadata_modified_at = datetime.utcnow()
|
|
134
142
|
dataservice.save()
|
|
135
|
-
|
|
136
143
|
return "", 204
|
|
137
144
|
|
|
138
145
|
|
udata/core/dataservices/apiv2.py
CHANGED
|
@@ -1,10 +1,7 @@
|
|
|
1
|
-
from flask import request
|
|
2
|
-
|
|
3
1
|
from udata import search
|
|
4
2
|
from udata.api import API, apiv2
|
|
5
3
|
from udata.core.access_type.models import AccessAudience
|
|
6
4
|
from udata.core.dataservices.models import Dataservice, HarvestMetadata
|
|
7
|
-
from udata.utils import multi_to_dict
|
|
8
5
|
|
|
9
6
|
from .models import dataservice_permissions_fields
|
|
10
7
|
from .search import DataserviceSearch
|
|
@@ -18,7 +15,7 @@ apiv2.inherit("AccessAudience (read)", AccessAudience.__read_fields__)
|
|
|
18
15
|
|
|
19
16
|
ns = apiv2.namespace("dataservices", "Dataservice related operations")
|
|
20
17
|
|
|
21
|
-
search_parser = DataserviceSearch.as_request_parser()
|
|
18
|
+
search_parser = DataserviceSearch.as_request_parser(store_missing=False)
|
|
22
19
|
|
|
23
20
|
|
|
24
21
|
@ns.route("/search/", endpoint="dataservice_search")
|
|
@@ -30,5 +27,5 @@ class DataserviceSearchAPI(API):
|
|
|
30
27
|
@apiv2.marshal_with(Dataservice.__page_fields__)
|
|
31
28
|
def get(self):
|
|
32
29
|
"""Search all dataservices"""
|
|
33
|
-
search_parser.parse_args()
|
|
34
|
-
return search.query(DataserviceSearch, **
|
|
30
|
+
args = search_parser.parse_args()
|
|
31
|
+
return search.query(DataserviceSearch, **args)
|
|
@@ -201,7 +201,10 @@ class Dataservice(
|
|
|
201
201
|
),
|
|
202
202
|
readonly=True,
|
|
203
203
|
)
|
|
204
|
-
description = field(
|
|
204
|
+
description = field(
|
|
205
|
+
db.StringField(default=""),
|
|
206
|
+
markdown=True,
|
|
207
|
+
)
|
|
205
208
|
base_api_url = field(db.URLField(), sortable=True)
|
|
206
209
|
|
|
207
210
|
machine_documentation_url = field(
|
|
@@ -309,7 +312,7 @@ class Dataservice(
|
|
|
309
312
|
|
|
310
313
|
@field(description="Link to the udata web page for this dataservice", show_as_ref=True)
|
|
311
314
|
def self_web_url(self, **kwargs):
|
|
312
|
-
return cdata_url(f"/dataservices/{self._link_id(**kwargs)}
|
|
315
|
+
return cdata_url(f"/dataservices/{self._link_id(**kwargs)}", **kwargs)
|
|
313
316
|
|
|
314
317
|
__metrics_keys__ = [
|
|
315
318
|
"discussions",
|
udata/core/dataservices/rdf.py
CHANGED
|
@@ -31,6 +31,7 @@ def dataservice_from_rdf(
|
|
|
31
31
|
node,
|
|
32
32
|
all_datasets: list[Dataset],
|
|
33
33
|
remote_url_prefix: str | None = None,
|
|
34
|
+
dryrun: bool = False,
|
|
34
35
|
) -> Dataservice:
|
|
35
36
|
"""
|
|
36
37
|
Create or update a dataservice from a RDF/DCAT graph
|
|
@@ -51,7 +52,7 @@ def dataservice_from_rdf(
|
|
|
51
52
|
dataservice.machine_documentation_url = url_from_rdf(d, DCAT.endpointDescription)
|
|
52
53
|
|
|
53
54
|
roles = [ # Imbricated list of contact points for each role
|
|
54
|
-
contact_points_from_rdf(d, rdf_entity, role, dataservice)
|
|
55
|
+
contact_points_from_rdf(d, rdf_entity, role, dataservice, dryrun=dryrun)
|
|
55
56
|
for rdf_entity, role in CONTACT_POINT_ENTITY_TO_ROLE.items()
|
|
56
57
|
]
|
|
57
58
|
dataservice.contact_points = [ # Flattened list of contact points
|