udata 14.0.0__py3-none-any.whl → 14.4.1.dev7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of udata might be problematic. Click here for more details.
- udata/api_fields.py +35 -4
- udata/app.py +18 -20
- udata/auth/__init__.py +29 -6
- udata/auth/forms.py +2 -2
- udata/auth/views.py +6 -3
- udata/commands/serve.py +3 -11
- udata/commands/tests/test_fixtures.py +9 -9
- udata/core/access_type/api.py +1 -1
- udata/core/access_type/constants.py +12 -8
- udata/core/activity/api.py +5 -6
- udata/core/badges/tests/test_commands.py +6 -6
- udata/core/csv.py +5 -0
- udata/core/dataservices/models.py +1 -1
- udata/core/dataservices/tasks.py +7 -0
- udata/core/dataset/api.py +2 -0
- udata/core/dataset/models.py +2 -2
- udata/core/dataset/permissions.py +31 -0
- udata/core/dataset/tasks.py +17 -5
- udata/core/discussions/models.py +1 -0
- udata/core/organization/api.py +8 -5
- udata/core/organization/mails.py +1 -1
- udata/core/organization/models.py +9 -1
- udata/core/organization/notifications.py +84 -0
- udata/core/organization/permissions.py +1 -1
- udata/core/organization/tasks.py +3 -0
- udata/core/pages/tests/test_api.py +32 -0
- udata/core/post/api.py +24 -69
- udata/core/post/models.py +84 -16
- udata/core/post/tests/test_api.py +24 -1
- udata/core/reports/api.py +18 -0
- udata/core/reports/models.py +42 -2
- udata/core/reuse/models.py +1 -1
- udata/core/reuse/tasks.py +7 -0
- udata/core/spatial/forms.py +2 -2
- udata/core/user/models.py +5 -1
- udata/features/notifications/api.py +7 -18
- udata/features/notifications/models.py +56 -0
- udata/features/notifications/tasks.py +25 -0
- udata/flask_mongoengine/engine.py +0 -4
- udata/frontend/markdown.py +2 -1
- udata/harvest/actions.py +21 -1
- udata/harvest/api.py +25 -8
- udata/harvest/backends/base.py +27 -1
- udata/harvest/backends/ckan/harvesters.py +11 -2
- udata/harvest/commands.py +33 -0
- udata/harvest/filters.py +17 -6
- udata/harvest/models.py +16 -0
- udata/harvest/permissions.py +27 -0
- udata/harvest/tests/ckan/test_ckan_backend.py +33 -0
- udata/harvest/tests/test_actions.py +58 -5
- udata/harvest/tests/test_api.py +276 -122
- udata/harvest/tests/test_base_backend.py +86 -1
- udata/harvest/tests/test_dcat_backend.py +57 -10
- udata/harvest/tests/test_filters.py +6 -0
- udata/i18n.py +1 -4
- udata/mail.py +5 -1
- udata/migrations/2025-10-31-create-membership-request-notifications.py +55 -0
- udata/migrations/2025-12-04-add-uuid-to-discussion-messages.py +28 -0
- udata/mongo/slug_fields.py +1 -1
- udata/rdf.py +45 -6
- udata/routing.py +2 -2
- udata/settings.py +7 -0
- udata/tasks.py +1 -0
- udata/templates/mail/message.html +5 -31
- udata/tests/__init__.py +27 -2
- udata/tests/api/__init__.py +108 -21
- udata/tests/api/test_activities_api.py +36 -0
- udata/tests/api/test_auth_api.py +121 -95
- udata/tests/api/test_base_api.py +7 -4
- udata/tests/api/test_datasets_api.py +44 -19
- udata/tests/api/test_organizations_api.py +192 -197
- udata/tests/api/test_reports_api.py +157 -0
- udata/tests/api/test_reuses_api.py +147 -147
- udata/tests/api/test_security_api.py +12 -12
- udata/tests/api/test_swagger.py +4 -4
- udata/tests/api/test_tags_api.py +8 -8
- udata/tests/api/test_user_api.py +1 -1
- udata/tests/apiv2/test_swagger.py +4 -4
- udata/tests/cli/test_cli_base.py +8 -9
- udata/tests/dataset/test_dataset_commands.py +4 -4
- udata/tests/dataset/test_dataset_model.py +66 -26
- udata/tests/dataset/test_dataset_rdf.py +99 -5
- udata/tests/frontend/test_auth.py +24 -1
- udata/tests/frontend/test_csv.py +0 -3
- udata/tests/helpers.py +25 -27
- udata/tests/organization/test_notifications.py +67 -2
- udata/tests/plugin.py +6 -261
- udata/tests/site/test_site_csv_exports.py +22 -10
- udata/tests/test_activity.py +9 -9
- udata/tests/test_dcat_commands.py +2 -2
- udata/tests/test_discussions.py +5 -5
- udata/tests/test_migrations.py +21 -21
- udata/tests/test_notifications.py +15 -57
- udata/tests/test_notifications_task.py +43 -0
- udata/tests/test_owned.py +81 -1
- udata/tests/test_storages.py +25 -19
- udata/tests/test_topics.py +77 -61
- udata/tests/test_uris.py +33 -0
- udata/tests/workers/test_jobs_commands.py +23 -23
- udata/translations/ar/LC_MESSAGES/udata.mo +0 -0
- udata/translations/ar/LC_MESSAGES/udata.po +187 -108
- udata/translations/de/LC_MESSAGES/udata.mo +0 -0
- udata/translations/de/LC_MESSAGES/udata.po +187 -108
- udata/translations/es/LC_MESSAGES/udata.mo +0 -0
- udata/translations/es/LC_MESSAGES/udata.po +187 -108
- udata/translations/fr/LC_MESSAGES/udata.mo +0 -0
- udata/translations/fr/LC_MESSAGES/udata.po +188 -109
- udata/translations/it/LC_MESSAGES/udata.mo +0 -0
- udata/translations/it/LC_MESSAGES/udata.po +187 -108
- udata/translations/pt/LC_MESSAGES/udata.mo +0 -0
- udata/translations/pt/LC_MESSAGES/udata.po +187 -108
- udata/translations/sr/LC_MESSAGES/udata.mo +0 -0
- udata/translations/sr/LC_MESSAGES/udata.po +187 -108
- udata/translations/udata.pot +215 -106
- udata/uris.py +0 -2
- udata-14.4.1.dev7.dist-info/METADATA +109 -0
- {udata-14.0.0.dist-info → udata-14.4.1.dev7.dist-info}/RECORD +121 -123
- udata/core/post/forms.py +0 -30
- udata/flask_mongoengine/json.py +0 -38
- udata/templates/mail/base.html +0 -105
- udata/templates/mail/base.txt +0 -6
- udata/templates/mail/button.html +0 -3
- udata/templates/mail/layouts/1-column.html +0 -19
- udata/templates/mail/layouts/2-columns.html +0 -20
- udata/templates/mail/layouts/center-panel.html +0 -16
- udata-14.0.0.dist-info/METADATA +0 -132
- {udata-14.0.0.dist-info → udata-14.4.1.dev7.dist-info}/WHEEL +0 -0
- {udata-14.0.0.dist-info → udata-14.4.1.dev7.dist-info}/entry_points.txt +0 -0
- {udata-14.0.0.dist-info → udata-14.4.1.dev7.dist-info}/licenses/LICENSE +0 -0
- {udata-14.0.0.dist-info → udata-14.4.1.dev7.dist-info}/top_level.txt +0 -0
udata/api_fields.py
CHANGED
|
@@ -29,7 +29,7 @@ import flask_restx.fields as restx_fields
|
|
|
29
29
|
import mongoengine
|
|
30
30
|
import mongoengine.fields as mongo_fields
|
|
31
31
|
from bson import DBRef, ObjectId
|
|
32
|
-
from flask import Request
|
|
32
|
+
from flask import Request, request
|
|
33
33
|
from flask_restx import marshal
|
|
34
34
|
from flask_restx.inputs import boolean
|
|
35
35
|
from flask_restx.reqparse import RequestParser
|
|
@@ -55,8 +55,8 @@ classes_by_parents = {}
|
|
|
55
55
|
|
|
56
56
|
|
|
57
57
|
class GenericField(restx_fields.Raw):
|
|
58
|
-
def __init__(self, fields_by_type):
|
|
59
|
-
super().__init__(
|
|
58
|
+
def __init__(self, fields_by_type, **kwargs):
|
|
59
|
+
super(GenericField, self).__init__(**kwargs)
|
|
60
60
|
self.default = None
|
|
61
61
|
self.fields_by_type = fields_by_type
|
|
62
62
|
|
|
@@ -230,6 +230,23 @@ def convert_db_to_field(key, field, info) -> tuple[Callable | None, Callable | N
|
|
|
230
230
|
|
|
231
231
|
write_params["description"] = "ID of the reference"
|
|
232
232
|
constructor_write = restx_fields.String
|
|
233
|
+
elif isinstance(field, mongo_fields.GenericEmbeddedDocumentField):
|
|
234
|
+
generic_fields = {
|
|
235
|
+
cls.__name__: convert_db_to_field(
|
|
236
|
+
f"{key}.{cls.__name__}",
|
|
237
|
+
# Instead of having GenericEmbeddedDocumentField() we'll create fields for each
|
|
238
|
+
# of the subclasses with EmbededdDocumentField(MembershipRequestNotificationDetails)…
|
|
239
|
+
mongoengine.fields.EmbeddedDocumentField(cls),
|
|
240
|
+
info,
|
|
241
|
+
)
|
|
242
|
+
for cls in field.choices
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
def constructor_read(**kwargs):
|
|
246
|
+
return GenericField({k: v[0].model for k, v in generic_fields.items()}, **kwargs)
|
|
247
|
+
|
|
248
|
+
def constructor_write(**kwargs):
|
|
249
|
+
return GenericField({k: v[1].model for k, v in generic_fields.items()}, **kwargs)
|
|
233
250
|
elif isinstance(field, mongo_fields.EmbeddedDocumentField):
|
|
234
251
|
nested_fields = info.get("nested_fields")
|
|
235
252
|
if nested_fields is not None:
|
|
@@ -322,12 +339,12 @@ def generate_fields(**kwargs) -> Callable:
|
|
|
322
339
|
write_fields: dict = {}
|
|
323
340
|
ref_fields: dict = {}
|
|
324
341
|
sortables: list = kwargs.get("additional_sorts", [])
|
|
342
|
+
default_sort: list = kwargs.get("default_sort", None)
|
|
325
343
|
|
|
326
344
|
filterables: list[dict] = kwargs.get("standalone_filters", [])
|
|
327
345
|
nested_filters: dict[str, dict] = get_fields_with_nested_filters(
|
|
328
346
|
kwargs.get("nested_filters", {})
|
|
329
347
|
)
|
|
330
|
-
|
|
331
348
|
if issubclass(cls, db.Document) or issubclass(cls, db.DynamicDocument):
|
|
332
349
|
read_fields["id"] = restx_fields.String(required=True, readonly=True)
|
|
333
350
|
|
|
@@ -472,6 +489,7 @@ def generate_fields(**kwargs) -> Callable:
|
|
|
472
489
|
type=str,
|
|
473
490
|
location="args",
|
|
474
491
|
choices=choices,
|
|
492
|
+
default=default_sort,
|
|
475
493
|
help="The field (and direction) on which sorting apply",
|
|
476
494
|
)
|
|
477
495
|
|
|
@@ -511,6 +529,9 @@ def generate_fields(**kwargs) -> Callable:
|
|
|
511
529
|
phrase_query: str = " ".join([f'"{elem}"' for elem in args["q"].split(" ")])
|
|
512
530
|
base_query = base_query.search_text(phrase_query)
|
|
513
531
|
|
|
532
|
+
if "sort" not in request.args:
|
|
533
|
+
base_query = base_query.order_by("$text_score")
|
|
534
|
+
|
|
514
535
|
for filterable in filterables:
|
|
515
536
|
# If it's from an `nested_filter`, use the custom label instead of the key,
|
|
516
537
|
# eg use `organization_badge` instead of `organization.badges` which is
|
|
@@ -652,6 +673,9 @@ def patch(obj, request) -> type:
|
|
|
652
673
|
model_attribute = getattr(obj.__class__, key)
|
|
653
674
|
info = getattr(model_attribute, "__additional_field_info__", {})
|
|
654
675
|
|
|
676
|
+
if value == "" and isinstance(model_attribute, mongo_fields.StringField):
|
|
677
|
+
value = None
|
|
678
|
+
|
|
655
679
|
if hasattr(model_attribute, "from_input"):
|
|
656
680
|
value = model_attribute.from_input(value)
|
|
657
681
|
elif isinstance(model_attribute, mongoengine.fields.ListField) and isinstance(
|
|
@@ -677,6 +701,13 @@ def patch(obj, request) -> type:
|
|
|
677
701
|
value["id"],
|
|
678
702
|
document_type=db.resolve_model(value["class"]),
|
|
679
703
|
)
|
|
704
|
+
elif value and isinstance(
|
|
705
|
+
model_attribute,
|
|
706
|
+
mongoengine.fields.GenericEmbeddedDocumentField,
|
|
707
|
+
):
|
|
708
|
+
generic_key = info.get("generic_key", DEFAULT_GENERIC_KEY)
|
|
709
|
+
embedded_field = classes_by_names[value[generic_key]]
|
|
710
|
+
value = patch(embedded_field(), value)
|
|
680
711
|
elif value and isinstance(
|
|
681
712
|
model_attribute,
|
|
682
713
|
mongoengine.fields.EmbeddedDocumentField,
|
udata/app.py
CHANGED
|
@@ -1,26 +1,28 @@
|
|
|
1
|
-
import datetime
|
|
2
1
|
import logging
|
|
3
2
|
import os
|
|
4
3
|
import types
|
|
4
|
+
from datetime import datetime
|
|
5
5
|
from importlib.metadata import entry_points
|
|
6
6
|
from os.path import abspath, dirname, exists, isfile, join
|
|
7
7
|
|
|
8
8
|
import bson
|
|
9
|
+
from bson import json_util
|
|
9
10
|
from flask import Blueprint as BaseBlueprint
|
|
10
11
|
from flask import (
|
|
11
12
|
Flask,
|
|
12
13
|
abort,
|
|
13
14
|
g,
|
|
14
|
-
json,
|
|
15
15
|
jsonify,
|
|
16
16
|
make_response,
|
|
17
17
|
render_template,
|
|
18
18
|
request,
|
|
19
19
|
send_from_directory,
|
|
20
20
|
)
|
|
21
|
+
from flask.json.provider import DefaultJSONProvider
|
|
21
22
|
from flask_caching import Cache
|
|
22
23
|
from flask_wtf.csrf import CSRFProtect
|
|
23
|
-
from
|
|
24
|
+
from mongoengine import EmbeddedDocument
|
|
25
|
+
from mongoengine.base import BaseDocument
|
|
24
26
|
from werkzeug.exceptions import NotFound
|
|
25
27
|
from werkzeug.middleware.proxy_fix import ProxyFix
|
|
26
28
|
|
|
@@ -108,9 +110,9 @@ class Blueprint(BaseBlueprint):
|
|
|
108
110
|
return wrapper
|
|
109
111
|
|
|
110
112
|
|
|
111
|
-
class
|
|
113
|
+
class UdataJsonProvider(DefaultJSONProvider):
|
|
112
114
|
"""
|
|
113
|
-
A
|
|
115
|
+
A JSONProvider subclass to encode unsupported types:
|
|
114
116
|
|
|
115
117
|
- ObjectId
|
|
116
118
|
- datetime
|
|
@@ -120,21 +122,16 @@ class UDataJsonEncoder(json.JSONEncoder):
|
|
|
120
122
|
Ensure an app context is always present.
|
|
121
123
|
"""
|
|
122
124
|
|
|
123
|
-
|
|
124
|
-
|
|
125
|
+
@staticmethod
|
|
126
|
+
def default(obj):
|
|
127
|
+
if isinstance(obj, BaseDocument) or isinstance(obj, EmbeddedDocument):
|
|
128
|
+
return json_util._json_convert(obj.to_mongo())
|
|
129
|
+
elif isinstance(obj, bson.ObjectId):
|
|
125
130
|
return str(obj)
|
|
126
|
-
elif isinstance(obj,
|
|
127
|
-
return str(obj)
|
|
128
|
-
elif isinstance(obj, datetime.datetime):
|
|
131
|
+
elif isinstance(obj, datetime):
|
|
129
132
|
return obj.isoformat()
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
elif hasattr(obj, "serialize"):
|
|
133
|
-
return obj.serialize()
|
|
134
|
-
# Serialize Raw data for Document and EmbeddedDocument.
|
|
135
|
-
elif hasattr(obj, "_data"):
|
|
136
|
-
return obj._data
|
|
137
|
-
return super(UDataJsonEncoder, self).default(obj)
|
|
133
|
+
|
|
134
|
+
return super(UdataJsonProvider, UdataJsonProvider).default(obj)
|
|
138
135
|
|
|
139
136
|
|
|
140
137
|
# These loggers are very verbose
|
|
@@ -166,10 +163,11 @@ def create_app(config="udata.settings.Defaults", override=None, init_logging=ini
|
|
|
166
163
|
if override:
|
|
167
164
|
app.config.from_object(override)
|
|
168
165
|
|
|
169
|
-
app.
|
|
166
|
+
app.json_provider_class = UdataJsonProvider
|
|
167
|
+
app.json = app.json_provider_class(app)
|
|
170
168
|
|
|
171
169
|
# `ujson` doesn't support `cls` parameter https://github.com/ultrajson/ultrajson/issues/124
|
|
172
|
-
app.config["RESTX_JSON"] = {"
|
|
170
|
+
app.config["RESTX_JSON"] = {"default": UdataJsonProvider.default}
|
|
173
171
|
|
|
174
172
|
app.debug = app.config["DEBUG"] and not app.config["TESTING"]
|
|
175
173
|
|
udata/auth/__init__.py
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import logging
|
|
2
|
+
import typing as t
|
|
2
3
|
|
|
3
4
|
from flask import render_template
|
|
4
5
|
from flask_principal import Permission as BasePermission
|
|
@@ -10,6 +11,7 @@ from flask_security import Security as Security
|
|
|
10
11
|
from flask_security import current_user as current_user
|
|
11
12
|
from flask_security import login_required as login_required
|
|
12
13
|
from flask_security import login_user as login_user
|
|
14
|
+
from flask_security import mail_util
|
|
13
15
|
|
|
14
16
|
from . import mails
|
|
15
17
|
|
|
@@ -24,9 +26,6 @@ def render_security_template(template_name_or_list, **kwargs):
|
|
|
24
26
|
return render_template(template_name_or_list, **kwargs)
|
|
25
27
|
|
|
26
28
|
|
|
27
|
-
security = Security()
|
|
28
|
-
|
|
29
|
-
|
|
30
29
|
class Permission(BasePermission):
|
|
31
30
|
def __init__(self, *needs):
|
|
32
31
|
"""Let administrator bypass all permissions"""
|
|
@@ -36,6 +35,24 @@ class Permission(BasePermission):
|
|
|
36
35
|
admin_permission = Permission()
|
|
37
36
|
|
|
38
37
|
|
|
38
|
+
class NoopMailUtil(mail_util.MailUtil):
|
|
39
|
+
def send_mail(
|
|
40
|
+
self,
|
|
41
|
+
template: str,
|
|
42
|
+
subject: str,
|
|
43
|
+
recipient: str,
|
|
44
|
+
sender: t.Union[str, tuple],
|
|
45
|
+
body: str,
|
|
46
|
+
html: t.Optional[str],
|
|
47
|
+
**kwargs: t.Any,
|
|
48
|
+
) -> None:
|
|
49
|
+
log.debug(f"Sending mail {subject} to {recipient}")
|
|
50
|
+
log.debug(body)
|
|
51
|
+
log.debug(html)
|
|
52
|
+
|
|
53
|
+
return None
|
|
54
|
+
|
|
55
|
+
|
|
39
56
|
def init_app(app):
|
|
40
57
|
from udata.models import datastore
|
|
41
58
|
|
|
@@ -61,19 +78,25 @@ def init_app(app):
|
|
|
61
78
|
app.config["CDATA_BASE_URL"] + "?flash=confirm_error",
|
|
62
79
|
)
|
|
63
80
|
|
|
64
|
-
|
|
65
|
-
|
|
81
|
+
# Same logic as in our own mail system :DisableMail
|
|
82
|
+
debug = app.config.get("DEBUG", False)
|
|
83
|
+
send_mail = app.config.get("SEND_MAIL", not debug)
|
|
84
|
+
mail_util_cls = mail_util.MailUtil if send_mail else NoopMailUtil
|
|
85
|
+
|
|
86
|
+
security = Security(
|
|
66
87
|
datastore,
|
|
67
88
|
register_blueprint=False,
|
|
68
89
|
render_template=render_security_template,
|
|
69
90
|
login_form=ExtendedLoginForm,
|
|
70
|
-
confirm_register_form=ExtendedRegisterForm,
|
|
71
91
|
register_form=ExtendedRegisterForm,
|
|
72
92
|
reset_password_form=ExtendedResetPasswordForm,
|
|
73
93
|
forgot_password_form=ExtendedForgotPasswordForm,
|
|
74
94
|
password_util_cls=UdataPasswordUtil,
|
|
95
|
+
mail_util_cls=mail_util_cls,
|
|
75
96
|
)
|
|
76
97
|
|
|
98
|
+
security.init_app(app, datastore, register_blueprint=False)
|
|
99
|
+
|
|
77
100
|
security_bp = create_security_blueprint(app, app.extensions["security"], "security_blueprint")
|
|
78
101
|
|
|
79
102
|
app.register_blueprint(security_bp)
|
udata/auth/forms.py
CHANGED
|
@@ -8,7 +8,7 @@ from flask_security.forms import (
|
|
|
8
8
|
ForgotPasswordForm,
|
|
9
9
|
Form,
|
|
10
10
|
LoginForm,
|
|
11
|
-
|
|
11
|
+
RegisterFormV2,
|
|
12
12
|
ResetPasswordForm,
|
|
13
13
|
)
|
|
14
14
|
|
|
@@ -31,7 +31,7 @@ class WithCaptcha:
|
|
|
31
31
|
return False
|
|
32
32
|
|
|
33
33
|
|
|
34
|
-
class ExtendedRegisterForm(WithCaptcha,
|
|
34
|
+
class ExtendedRegisterForm(WithCaptcha, RegisterFormV2):
|
|
35
35
|
first_name = fields.StringField(
|
|
36
36
|
_("First name"),
|
|
37
37
|
[
|
udata/auth/views.py
CHANGED
|
@@ -84,6 +84,11 @@ def confirm_change_email(token):
|
|
|
84
84
|
if flash:
|
|
85
85
|
return redirect(homepage_url(flash=flash, flash_data=flash_data))
|
|
86
86
|
|
|
87
|
+
# Check if the new email is already taken by another user
|
|
88
|
+
existing_user = _datastore.find_user(email=new_email)
|
|
89
|
+
if existing_user and existing_user.id != user.id:
|
|
90
|
+
return redirect(homepage_url(flash="change_email_already_taken"))
|
|
91
|
+
|
|
87
92
|
if user != current_user:
|
|
88
93
|
logout_user()
|
|
89
94
|
login_user(user)
|
|
@@ -140,10 +145,8 @@ def create_security_blueprint(app, state, import_name):
|
|
|
140
145
|
This creates an I18nBlueprint to use as a base.
|
|
141
146
|
"""
|
|
142
147
|
bp = I18nBlueprint(
|
|
143
|
-
|
|
148
|
+
"security",
|
|
144
149
|
import_name,
|
|
145
|
-
url_prefix=state.url_prefix,
|
|
146
|
-
subdomain=state.subdomain,
|
|
147
150
|
template_folder="templates",
|
|
148
151
|
)
|
|
149
152
|
|
udata/commands/serve.py
CHANGED
|
@@ -3,7 +3,7 @@ import os
|
|
|
3
3
|
|
|
4
4
|
import click
|
|
5
5
|
from flask import current_app
|
|
6
|
-
from flask.cli import
|
|
6
|
+
from flask.cli import pass_script_info
|
|
7
7
|
from werkzeug.serving import run_simple
|
|
8
8
|
|
|
9
9
|
from udata.commands import cli
|
|
@@ -26,17 +26,11 @@ log = logging.getLogger(__name__)
|
|
|
26
26
|
default=None,
|
|
27
27
|
help="Enable or disable the debugger. By default the debugger is active if debug is enabled.",
|
|
28
28
|
)
|
|
29
|
-
@click.option(
|
|
30
|
-
"--eager-loading/--lazy-loader",
|
|
31
|
-
default=None,
|
|
32
|
-
help="Enable or disable eager loading. By default eager "
|
|
33
|
-
"loading is enabled if the reloader is disabled.",
|
|
34
|
-
)
|
|
35
29
|
@click.option(
|
|
36
30
|
"--with-threads/--without-threads", default=True, help="Enable or disable multithreading."
|
|
37
31
|
)
|
|
38
32
|
@pass_script_info
|
|
39
|
-
def serve(info, host, port, reload, debugger,
|
|
33
|
+
def serve(info, host, port, reload, debugger, with_threads):
|
|
40
34
|
"""
|
|
41
35
|
Runs a local udata development server.
|
|
42
36
|
|
|
@@ -62,10 +56,8 @@ def serve(info, host, port, reload, debugger, eager_loading, with_threads):
|
|
|
62
56
|
reload = bool(debug)
|
|
63
57
|
if debugger is None:
|
|
64
58
|
debugger = bool(debug)
|
|
65
|
-
if eager_loading is None:
|
|
66
|
-
eager_loading = not reload
|
|
67
59
|
|
|
68
|
-
app =
|
|
60
|
+
app = info.load_app()
|
|
69
61
|
|
|
70
62
|
settings = os.environ.get("UDATA_SETTINGS", os.path.join(os.getcwd(), "udata.cfg"))
|
|
71
63
|
extra_files = [settings]
|
|
@@ -24,7 +24,7 @@ from udata.tests.api import PytestOnlyAPITestCase
|
|
|
24
24
|
|
|
25
25
|
class FixturesTest(PytestOnlyAPITestCase):
|
|
26
26
|
@pytest.mark.options(FIXTURE_DATASET_SLUGS=["some-test-dataset-slug"])
|
|
27
|
-
def test_generate_fixtures_file_then_import(self,
|
|
27
|
+
def test_generate_fixtures_file_then_import(self, mocker):
|
|
28
28
|
"""Test generating fixtures from the current env, then importing them back."""
|
|
29
29
|
assert models.Dataset.objects.count() == 0 # Start with a clean slate.
|
|
30
30
|
user = UserFactory()
|
|
@@ -55,11 +55,11 @@ class FixturesTest(PytestOnlyAPITestCase):
|
|
|
55
55
|
DataserviceFactory(datasets=[dataset], organization=org, contact_points=[contact_point])
|
|
56
56
|
|
|
57
57
|
with NamedTemporaryFile(mode="w+", delete=True) as fixtures_fd:
|
|
58
|
-
# Get the fixtures from the local instance.
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
Response
|
|
62
|
-
result = cli("generate-fixtures-file", "", fixtures_fd.name)
|
|
58
|
+
# Get the fixtures from the local instance by redirecting requests.get to the test client
|
|
59
|
+
mocker.patch.object(requests, "get", side_effect=lambda url: self.get(url))
|
|
60
|
+
mocker.patch.object(Response, "json", Response.get_json)
|
|
61
|
+
mocker.patch.object(Response, "ok", True, create=True)
|
|
62
|
+
result = self.cli("generate-fixtures-file", "", fixtures_fd.name)
|
|
63
63
|
fixtures_fd.flush()
|
|
64
64
|
assert "Fixtures saved to file " in result.output
|
|
65
65
|
|
|
@@ -81,7 +81,7 @@ class FixturesTest(PytestOnlyAPITestCase):
|
|
|
81
81
|
assert models.ContactPoint.objects.count() == 0
|
|
82
82
|
|
|
83
83
|
# Then load them in the database to make sure they're correct.
|
|
84
|
-
result = cli("import-fixtures", fixtures_fd.name)
|
|
84
|
+
result = self.cli("import-fixtures", fixtures_fd.name)
|
|
85
85
|
assert models.Organization.objects(slug=org.slug).count() > 0
|
|
86
86
|
result_org = models.Organization.objects.get(slug=org.slug)
|
|
87
87
|
assert result_org.members[0].user.id == user.id
|
|
@@ -106,11 +106,11 @@ class FixturesTest(PytestOnlyAPITestCase):
|
|
|
106
106
|
assert result_dataservice.organization == org
|
|
107
107
|
assert result_dataservice.contact_points == [contact_point]
|
|
108
108
|
|
|
109
|
-
def test_import_fixtures_from_default_file(self
|
|
109
|
+
def test_import_fixtures_from_default_file(self):
|
|
110
110
|
"""Test importing fixtures from udata.commands.fixture.DEFAULT_FIXTURE_FILE."""
|
|
111
111
|
# Deactivate spam detection when testing import fixtures
|
|
112
112
|
SpamMixin.detect_spam_enabled = False
|
|
113
|
-
cli("import-fixtures")
|
|
113
|
+
self.cli("import-fixtures")
|
|
114
114
|
SpamMixin.detect_spam_enabled = True
|
|
115
115
|
assert models.Organization.objects.count() > 0
|
|
116
116
|
assert models.Dataset.objects.count() > 0
|
udata/core/access_type/api.py
CHANGED
|
@@ -13,6 +13,6 @@ class ReasonCategoriesAPI(API):
|
|
|
13
13
|
def get(self):
|
|
14
14
|
"""List all limitation reason categories"""
|
|
15
15
|
return [
|
|
16
|
-
{"value": category.value, "label": category.label}
|
|
16
|
+
{"value": category.value, "label": category.label, "definition": category.definition}
|
|
17
17
|
for category in InspireLimitationCategory
|
|
18
18
|
]
|
|
@@ -63,31 +63,35 @@ class InspireLimitationCategory(StrEnum):
|
|
|
63
63
|
match self:
|
|
64
64
|
case InspireLimitationCategory.PUBLIC_AUTHORITIES:
|
|
65
65
|
return _(
|
|
66
|
-
"
|
|
66
|
+
"Public access to datasets and services would adversely affect the confidentiality of the proceedings of public authorities, where such confidentiality is provided for by law."
|
|
67
67
|
)
|
|
68
68
|
case InspireLimitationCategory.INTERNATIONAL_RELATIONS:
|
|
69
|
-
return _(
|
|
69
|
+
return _(
|
|
70
|
+
"Public access to datasets and services would adversely affect international relations, public security or national defence."
|
|
71
|
+
)
|
|
70
72
|
case InspireLimitationCategory.COURSE_OF_JUSTICE:
|
|
71
73
|
return _(
|
|
72
|
-
"
|
|
74
|
+
"Public access to datasets and services would adversely affect the course of justice, the ability of any person to receive a fair trial or the ability of a public authority to conduct an enquiry of a criminal or disciplinary nature."
|
|
73
75
|
)
|
|
74
76
|
case InspireLimitationCategory.COMMERCIAL_CONFIDENTIALITY:
|
|
75
77
|
return _(
|
|
76
|
-
"
|
|
78
|
+
"Public access to datasets and services would adversely affect the confidentiality of commercial or industrial information, where such confidentiality is provided for by national or Community law to protect a legitimate economic interest, including the public interest in maintaining statistical confidentiality and tax secrecy."
|
|
77
79
|
)
|
|
78
80
|
case InspireLimitationCategory.INTELLECTUAL_PROPERTY:
|
|
79
|
-
return _(
|
|
81
|
+
return _(
|
|
82
|
+
"Public access to datasets and services would adversely affect intellectual property rights."
|
|
83
|
+
)
|
|
80
84
|
case InspireLimitationCategory.PERSONAL_DATA:
|
|
81
85
|
return _(
|
|
82
|
-
"
|
|
86
|
+
"Public access to datasets and services would adversely affect the confidentiality of personal data and/or files relating to a natural person where that person has not consented to the disclosure of the information to the public, where such confidentiality is provided for by national or Community law."
|
|
83
87
|
)
|
|
84
88
|
case InspireLimitationCategory.VOLUNTARY_SUPPLIER:
|
|
85
89
|
return _(
|
|
86
|
-
"
|
|
90
|
+
"Public access to datasets and services would adversely affect the interests or protection of any person who supplied the information requested on a voluntary basis without being under, or capable of being put under, a legal obligation to do so, unless that person has consented to the release of the information concerned."
|
|
87
91
|
)
|
|
88
92
|
case InspireLimitationCategory.ENVIRONMENTAL_PROTECTION:
|
|
89
93
|
return _(
|
|
90
|
-
"
|
|
94
|
+
"Public access to datasets and services would adversely affect the protection of the environment to which such information relates, such as the location of rare species."
|
|
91
95
|
)
|
|
92
96
|
case _:
|
|
93
97
|
assert_never(self)
|
udata/core/activity/api.py
CHANGED
|
@@ -4,8 +4,9 @@ from bson import ObjectId
|
|
|
4
4
|
from mongoengine.errors import DoesNotExist
|
|
5
5
|
|
|
6
6
|
from udata.api import API, api, fields
|
|
7
|
-
from udata.
|
|
7
|
+
from udata.core.dataset.permissions import OwnableReadPermission
|
|
8
8
|
from udata.core.organization.api_fields import org_ref_fields
|
|
9
|
+
from udata.core.owned import Owned
|
|
9
10
|
from udata.core.user.api_fields import user_ref_fields
|
|
10
11
|
from udata.models import Activity, db
|
|
11
12
|
|
|
@@ -101,7 +102,7 @@ class SiteActivityAPI(API):
|
|
|
101
102
|
# Always return a result even not complete
|
|
102
103
|
# But log the error (ie. visible in sentry, silent for user)
|
|
103
104
|
# Can happen when someone manually delete an object in DB (ie. without proper purge)
|
|
104
|
-
# - Filter out
|
|
105
|
+
# - Filter out items not visible to the current user
|
|
105
106
|
safe_items = []
|
|
106
107
|
for item in qs.queryset.items:
|
|
107
108
|
try:
|
|
@@ -109,10 +110,8 @@ class SiteActivityAPI(API):
|
|
|
109
110
|
except DoesNotExist as e:
|
|
110
111
|
log.error(e, exc_info=True)
|
|
111
112
|
else:
|
|
112
|
-
if
|
|
113
|
-
|
|
114
|
-
):
|
|
115
|
-
if item.related_to.private:
|
|
113
|
+
if isinstance(item.related_to, Owned):
|
|
114
|
+
if not OwnableReadPermission(item.related_to).can():
|
|
116
115
|
continue
|
|
117
116
|
safe_items.append(item)
|
|
118
117
|
qs.queryset.items = safe_items
|
|
@@ -9,32 +9,32 @@ class BadgeCommandTest(PytestOnlyDBTestCase):
|
|
|
9
9
|
def toggle(self, path_or_id, kind):
|
|
10
10
|
return self.cli("badges", "toggle", path_or_id, kind)
|
|
11
11
|
|
|
12
|
-
def test_toggle_badge_on(self
|
|
12
|
+
def test_toggle_badge_on(self):
|
|
13
13
|
org = OrganizationFactory()
|
|
14
14
|
|
|
15
|
-
cli("badges", "toggle", str(org.id), PUBLIC_SERVICE)
|
|
15
|
+
self.cli("badges", "toggle", str(org.id), PUBLIC_SERVICE)
|
|
16
16
|
|
|
17
17
|
org.reload()
|
|
18
18
|
assert org.badges[0].kind == PUBLIC_SERVICE
|
|
19
19
|
|
|
20
|
-
def test_toggle_badge_off(self
|
|
20
|
+
def test_toggle_badge_off(self):
|
|
21
21
|
org = OrganizationFactory()
|
|
22
22
|
org.add_badge(PUBLIC_SERVICE)
|
|
23
23
|
org.add_badge(CERTIFIED)
|
|
24
24
|
|
|
25
|
-
cli("badges", "toggle", str(org.id), PUBLIC_SERVICE)
|
|
25
|
+
self.cli("badges", "toggle", str(org.id), PUBLIC_SERVICE)
|
|
26
26
|
|
|
27
27
|
org.reload()
|
|
28
28
|
assert org.badges[0].kind == CERTIFIED
|
|
29
29
|
|
|
30
|
-
def test_toggle_badge_on_from_file(self
|
|
30
|
+
def test_toggle_badge_on_from_file(self):
|
|
31
31
|
orgs = [OrganizationFactory() for _ in range(2)]
|
|
32
32
|
|
|
33
33
|
with NamedTemporaryFile(mode="w") as temp:
|
|
34
34
|
temp.write("\n".join((str(org.id) for org in orgs)))
|
|
35
35
|
temp.flush()
|
|
36
36
|
|
|
37
|
-
cli("badges", "toggle", temp.name, PUBLIC_SERVICE)
|
|
37
|
+
self.cli("badges", "toggle", temp.name, PUBLIC_SERVICE)
|
|
38
38
|
|
|
39
39
|
for org in orgs:
|
|
40
40
|
org.reload()
|
udata/core/csv.py
CHANGED
|
@@ -5,6 +5,7 @@ from datetime import date, datetime
|
|
|
5
5
|
from io import StringIO
|
|
6
6
|
|
|
7
7
|
from flask import Response, stream_with_context
|
|
8
|
+
from mongoengine.queryset import QuerySet
|
|
8
9
|
|
|
9
10
|
from udata.mongo import db
|
|
10
11
|
from udata.utils import recursive_get
|
|
@@ -35,6 +36,10 @@ class Adapter(object):
|
|
|
35
36
|
fields = None
|
|
36
37
|
|
|
37
38
|
def __init__(self, queryset):
|
|
39
|
+
# no_cache() to avoid eating up too much RAM when iterating over large querysets.
|
|
40
|
+
# Applied here rather than upstream to preserve custom QuerySet methods (like with_badge).
|
|
41
|
+
if isinstance(queryset, QuerySet):
|
|
42
|
+
queryset = queryset.no_cache()
|
|
38
43
|
self.queryset = queryset
|
|
39
44
|
self._fields = None
|
|
40
45
|
|
|
@@ -309,7 +309,7 @@ class Dataservice(
|
|
|
309
309
|
|
|
310
310
|
@field(description="Link to the udata web page for this dataservice", show_as_ref=True)
|
|
311
311
|
def self_web_url(self, **kwargs):
|
|
312
|
-
return cdata_url(f"/dataservices/{self._link_id(**kwargs)}
|
|
312
|
+
return cdata_url(f"/dataservices/{self._link_id(**kwargs)}", **kwargs)
|
|
313
313
|
|
|
314
314
|
__metrics_keys__ = [
|
|
315
315
|
"discussions",
|
udata/core/dataservices/tasks.py
CHANGED
|
@@ -6,6 +6,7 @@ from udata.core.constants import HVD
|
|
|
6
6
|
from udata.core.dataservices.models import Dataservice
|
|
7
7
|
from udata.core.organization.constants import CERTIFIED, PUBLIC_SERVICE
|
|
8
8
|
from udata.core.organization.models import Organization
|
|
9
|
+
from udata.core.pages.models import Page
|
|
9
10
|
from udata.core.topic.models import TopicElement
|
|
10
11
|
from udata.harvest.models import HarvestJob
|
|
11
12
|
from udata.models import Discussion, Follow, Transfer
|
|
@@ -28,6 +29,12 @@ def purge_dataservices(self):
|
|
|
28
29
|
Transfer.objects(subject=dataservice).delete()
|
|
29
30
|
# Remove dataservices references in Topics
|
|
30
31
|
TopicElement.objects(element=dataservice).update(element=None)
|
|
32
|
+
# Remove dataservices in pages (mongoengine doesn't support updating a field in a generic embed)
|
|
33
|
+
Page._get_collection().update_many(
|
|
34
|
+
{"blocs.dataservices": dataservice.id},
|
|
35
|
+
{"$pull": {"blocs.$[b].dataservices": dataservice.id}},
|
|
36
|
+
array_filters=[{"b.dataservices": dataservice.id}],
|
|
37
|
+
)
|
|
31
38
|
# Remove dataservice
|
|
32
39
|
dataservice.delete()
|
|
33
40
|
|
udata/core/dataset/api.py
CHANGED
|
@@ -531,6 +531,8 @@ class ResourcesAPI(API):
|
|
|
531
531
|
f"All resources must be reordered, you provided {len(resources)} "
|
|
532
532
|
f"out of {len(dataset.resources)}",
|
|
533
533
|
)
|
|
534
|
+
if any(isinstance(r, dict) and "id" not in r for r in resources):
|
|
535
|
+
api.abort(400, "Each resource must have an 'id' field")
|
|
534
536
|
if set(r["id"] if isinstance(r, dict) else r for r in resources) != set(
|
|
535
537
|
str(r.id) for r in dataset.resources
|
|
536
538
|
):
|
udata/core/dataset/models.py
CHANGED
|
@@ -730,7 +730,7 @@ class Dataset(
|
|
|
730
730
|
}
|
|
731
731
|
|
|
732
732
|
def self_web_url(self, **kwargs):
|
|
733
|
-
return cdata_url(f"/datasets/{self._link_id(**kwargs)}
|
|
733
|
+
return cdata_url(f"/datasets/{self._link_id(**kwargs)}", **kwargs)
|
|
734
734
|
|
|
735
735
|
def self_api_url(self, **kwargs):
|
|
736
736
|
return url_for(
|
|
@@ -795,7 +795,7 @@ class Dataset(
|
|
|
795
795
|
Resources should be fetched when calling this method.
|
|
796
796
|
"""
|
|
797
797
|
if self.harvest and self.harvest.modified_at:
|
|
798
|
-
return self.harvest.modified_at
|
|
798
|
+
return to_naive_datetime(self.harvest.modified_at)
|
|
799
799
|
if self.resources:
|
|
800
800
|
return max([res.last_modified for res in self.resources])
|
|
801
801
|
else:
|
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
from flask_principal import Permission as BasePermission
|
|
2
|
+
from flask_principal import RoleNeed
|
|
3
|
+
|
|
1
4
|
from udata.auth import Permission, UserNeed
|
|
2
5
|
from udata.core.organization.permissions import (
|
|
3
6
|
OrganizationAdminNeed,
|
|
@@ -22,6 +25,34 @@ class OwnablePermission(Permission):
|
|
|
22
25
|
super(OwnablePermission, self).__init__(*needs)
|
|
23
26
|
|
|
24
27
|
|
|
28
|
+
class OwnableReadPermission(BasePermission):
|
|
29
|
+
"""Permission to read a private ownable object.
|
|
30
|
+
|
|
31
|
+
Always grants access if the object is not private.
|
|
32
|
+
For private objects, requires owner, org member, or sysadmin.
|
|
33
|
+
|
|
34
|
+
We inherit from BasePermission instead of udata's Permission because
|
|
35
|
+
Permission automatically adds RoleNeed("admin") to all needs. This means
|
|
36
|
+
a permission with no needs would only allow admins. With BasePermission,
|
|
37
|
+
an empty needs set allows everyone (Flask-Principal returns True when
|
|
38
|
+
self.needs is empty).
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
def __init__(self, obj):
|
|
42
|
+
if not getattr(obj, "private", False):
|
|
43
|
+
super().__init__()
|
|
44
|
+
return
|
|
45
|
+
|
|
46
|
+
needs = [RoleNeed("admin")]
|
|
47
|
+
if obj.organization:
|
|
48
|
+
needs.append(OrganizationAdminNeed(obj.organization.id))
|
|
49
|
+
needs.append(OrganizationEditorNeed(obj.organization.id))
|
|
50
|
+
elif obj.owner:
|
|
51
|
+
needs.append(UserNeed(obj.owner.fs_uniquifier))
|
|
52
|
+
|
|
53
|
+
super().__init__(*needs)
|
|
54
|
+
|
|
55
|
+
|
|
25
56
|
class DatasetEditPermission(OwnablePermission):
|
|
26
57
|
"""Permissions to edit a Dataset"""
|
|
27
58
|
|