udata 14.0.0__py3-none-any.whl → 14.4.1.dev7__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of udata might be problematic. Click here for more details.

Files changed (130) hide show
  1. udata/api_fields.py +35 -4
  2. udata/app.py +18 -20
  3. udata/auth/__init__.py +29 -6
  4. udata/auth/forms.py +2 -2
  5. udata/auth/views.py +6 -3
  6. udata/commands/serve.py +3 -11
  7. udata/commands/tests/test_fixtures.py +9 -9
  8. udata/core/access_type/api.py +1 -1
  9. udata/core/access_type/constants.py +12 -8
  10. udata/core/activity/api.py +5 -6
  11. udata/core/badges/tests/test_commands.py +6 -6
  12. udata/core/csv.py +5 -0
  13. udata/core/dataservices/models.py +1 -1
  14. udata/core/dataservices/tasks.py +7 -0
  15. udata/core/dataset/api.py +2 -0
  16. udata/core/dataset/models.py +2 -2
  17. udata/core/dataset/permissions.py +31 -0
  18. udata/core/dataset/tasks.py +17 -5
  19. udata/core/discussions/models.py +1 -0
  20. udata/core/organization/api.py +8 -5
  21. udata/core/organization/mails.py +1 -1
  22. udata/core/organization/models.py +9 -1
  23. udata/core/organization/notifications.py +84 -0
  24. udata/core/organization/permissions.py +1 -1
  25. udata/core/organization/tasks.py +3 -0
  26. udata/core/pages/tests/test_api.py +32 -0
  27. udata/core/post/api.py +24 -69
  28. udata/core/post/models.py +84 -16
  29. udata/core/post/tests/test_api.py +24 -1
  30. udata/core/reports/api.py +18 -0
  31. udata/core/reports/models.py +42 -2
  32. udata/core/reuse/models.py +1 -1
  33. udata/core/reuse/tasks.py +7 -0
  34. udata/core/spatial/forms.py +2 -2
  35. udata/core/user/models.py +5 -1
  36. udata/features/notifications/api.py +7 -18
  37. udata/features/notifications/models.py +56 -0
  38. udata/features/notifications/tasks.py +25 -0
  39. udata/flask_mongoengine/engine.py +0 -4
  40. udata/frontend/markdown.py +2 -1
  41. udata/harvest/actions.py +21 -1
  42. udata/harvest/api.py +25 -8
  43. udata/harvest/backends/base.py +27 -1
  44. udata/harvest/backends/ckan/harvesters.py +11 -2
  45. udata/harvest/commands.py +33 -0
  46. udata/harvest/filters.py +17 -6
  47. udata/harvest/models.py +16 -0
  48. udata/harvest/permissions.py +27 -0
  49. udata/harvest/tests/ckan/test_ckan_backend.py +33 -0
  50. udata/harvest/tests/test_actions.py +58 -5
  51. udata/harvest/tests/test_api.py +276 -122
  52. udata/harvest/tests/test_base_backend.py +86 -1
  53. udata/harvest/tests/test_dcat_backend.py +57 -10
  54. udata/harvest/tests/test_filters.py +6 -0
  55. udata/i18n.py +1 -4
  56. udata/mail.py +5 -1
  57. udata/migrations/2025-10-31-create-membership-request-notifications.py +55 -0
  58. udata/migrations/2025-12-04-add-uuid-to-discussion-messages.py +28 -0
  59. udata/mongo/slug_fields.py +1 -1
  60. udata/rdf.py +45 -6
  61. udata/routing.py +2 -2
  62. udata/settings.py +7 -0
  63. udata/tasks.py +1 -0
  64. udata/templates/mail/message.html +5 -31
  65. udata/tests/__init__.py +27 -2
  66. udata/tests/api/__init__.py +108 -21
  67. udata/tests/api/test_activities_api.py +36 -0
  68. udata/tests/api/test_auth_api.py +121 -95
  69. udata/tests/api/test_base_api.py +7 -4
  70. udata/tests/api/test_datasets_api.py +44 -19
  71. udata/tests/api/test_organizations_api.py +192 -197
  72. udata/tests/api/test_reports_api.py +157 -0
  73. udata/tests/api/test_reuses_api.py +147 -147
  74. udata/tests/api/test_security_api.py +12 -12
  75. udata/tests/api/test_swagger.py +4 -4
  76. udata/tests/api/test_tags_api.py +8 -8
  77. udata/tests/api/test_user_api.py +1 -1
  78. udata/tests/apiv2/test_swagger.py +4 -4
  79. udata/tests/cli/test_cli_base.py +8 -9
  80. udata/tests/dataset/test_dataset_commands.py +4 -4
  81. udata/tests/dataset/test_dataset_model.py +66 -26
  82. udata/tests/dataset/test_dataset_rdf.py +99 -5
  83. udata/tests/frontend/test_auth.py +24 -1
  84. udata/tests/frontend/test_csv.py +0 -3
  85. udata/tests/helpers.py +25 -27
  86. udata/tests/organization/test_notifications.py +67 -2
  87. udata/tests/plugin.py +6 -261
  88. udata/tests/site/test_site_csv_exports.py +22 -10
  89. udata/tests/test_activity.py +9 -9
  90. udata/tests/test_dcat_commands.py +2 -2
  91. udata/tests/test_discussions.py +5 -5
  92. udata/tests/test_migrations.py +21 -21
  93. udata/tests/test_notifications.py +15 -57
  94. udata/tests/test_notifications_task.py +43 -0
  95. udata/tests/test_owned.py +81 -1
  96. udata/tests/test_storages.py +25 -19
  97. udata/tests/test_topics.py +77 -61
  98. udata/tests/test_uris.py +33 -0
  99. udata/tests/workers/test_jobs_commands.py +23 -23
  100. udata/translations/ar/LC_MESSAGES/udata.mo +0 -0
  101. udata/translations/ar/LC_MESSAGES/udata.po +187 -108
  102. udata/translations/de/LC_MESSAGES/udata.mo +0 -0
  103. udata/translations/de/LC_MESSAGES/udata.po +187 -108
  104. udata/translations/es/LC_MESSAGES/udata.mo +0 -0
  105. udata/translations/es/LC_MESSAGES/udata.po +187 -108
  106. udata/translations/fr/LC_MESSAGES/udata.mo +0 -0
  107. udata/translations/fr/LC_MESSAGES/udata.po +188 -109
  108. udata/translations/it/LC_MESSAGES/udata.mo +0 -0
  109. udata/translations/it/LC_MESSAGES/udata.po +187 -108
  110. udata/translations/pt/LC_MESSAGES/udata.mo +0 -0
  111. udata/translations/pt/LC_MESSAGES/udata.po +187 -108
  112. udata/translations/sr/LC_MESSAGES/udata.mo +0 -0
  113. udata/translations/sr/LC_MESSAGES/udata.po +187 -108
  114. udata/translations/udata.pot +215 -106
  115. udata/uris.py +0 -2
  116. udata-14.4.1.dev7.dist-info/METADATA +109 -0
  117. {udata-14.0.0.dist-info → udata-14.4.1.dev7.dist-info}/RECORD +121 -123
  118. udata/core/post/forms.py +0 -30
  119. udata/flask_mongoengine/json.py +0 -38
  120. udata/templates/mail/base.html +0 -105
  121. udata/templates/mail/base.txt +0 -6
  122. udata/templates/mail/button.html +0 -3
  123. udata/templates/mail/layouts/1-column.html +0 -19
  124. udata/templates/mail/layouts/2-columns.html +0 -20
  125. udata/templates/mail/layouts/center-panel.html +0 -16
  126. udata-14.0.0.dist-info/METADATA +0 -132
  127. {udata-14.0.0.dist-info → udata-14.4.1.dev7.dist-info}/WHEEL +0 -0
  128. {udata-14.0.0.dist-info → udata-14.4.1.dev7.dist-info}/entry_points.txt +0 -0
  129. {udata-14.0.0.dist-info → udata-14.4.1.dev7.dist-info}/licenses/LICENSE +0 -0
  130. {udata-14.0.0.dist-info → udata-14.4.1.dev7.dist-info}/top_level.txt +0 -0
udata/api_fields.py CHANGED
@@ -29,7 +29,7 @@ import flask_restx.fields as restx_fields
29
29
  import mongoengine
30
30
  import mongoengine.fields as mongo_fields
31
31
  from bson import DBRef, ObjectId
32
- from flask import Request
32
+ from flask import Request, request
33
33
  from flask_restx import marshal
34
34
  from flask_restx.inputs import boolean
35
35
  from flask_restx.reqparse import RequestParser
@@ -55,8 +55,8 @@ classes_by_parents = {}
55
55
 
56
56
 
57
57
  class GenericField(restx_fields.Raw):
58
- def __init__(self, fields_by_type):
59
- super().__init__(self)
58
+ def __init__(self, fields_by_type, **kwargs):
59
+ super(GenericField, self).__init__(**kwargs)
60
60
  self.default = None
61
61
  self.fields_by_type = fields_by_type
62
62
 
@@ -230,6 +230,23 @@ def convert_db_to_field(key, field, info) -> tuple[Callable | None, Callable | N
230
230
 
231
231
  write_params["description"] = "ID of the reference"
232
232
  constructor_write = restx_fields.String
233
+ elif isinstance(field, mongo_fields.GenericEmbeddedDocumentField):
234
+ generic_fields = {
235
+ cls.__name__: convert_db_to_field(
236
+ f"{key}.{cls.__name__}",
237
+ # Instead of having GenericEmbeddedDocumentField() we'll create fields for each
238
+ # of the subclasses with EmbededdDocumentField(MembershipRequestNotificationDetails)…
239
+ mongoengine.fields.EmbeddedDocumentField(cls),
240
+ info,
241
+ )
242
+ for cls in field.choices
243
+ }
244
+
245
+ def constructor_read(**kwargs):
246
+ return GenericField({k: v[0].model for k, v in generic_fields.items()}, **kwargs)
247
+
248
+ def constructor_write(**kwargs):
249
+ return GenericField({k: v[1].model for k, v in generic_fields.items()}, **kwargs)
233
250
  elif isinstance(field, mongo_fields.EmbeddedDocumentField):
234
251
  nested_fields = info.get("nested_fields")
235
252
  if nested_fields is not None:
@@ -322,12 +339,12 @@ def generate_fields(**kwargs) -> Callable:
322
339
  write_fields: dict = {}
323
340
  ref_fields: dict = {}
324
341
  sortables: list = kwargs.get("additional_sorts", [])
342
+ default_sort: list = kwargs.get("default_sort", None)
325
343
 
326
344
  filterables: list[dict] = kwargs.get("standalone_filters", [])
327
345
  nested_filters: dict[str, dict] = get_fields_with_nested_filters(
328
346
  kwargs.get("nested_filters", {})
329
347
  )
330
-
331
348
  if issubclass(cls, db.Document) or issubclass(cls, db.DynamicDocument):
332
349
  read_fields["id"] = restx_fields.String(required=True, readonly=True)
333
350
 
@@ -472,6 +489,7 @@ def generate_fields(**kwargs) -> Callable:
472
489
  type=str,
473
490
  location="args",
474
491
  choices=choices,
492
+ default=default_sort,
475
493
  help="The field (and direction) on which sorting apply",
476
494
  )
477
495
 
@@ -511,6 +529,9 @@ def generate_fields(**kwargs) -> Callable:
511
529
  phrase_query: str = " ".join([f'"{elem}"' for elem in args["q"].split(" ")])
512
530
  base_query = base_query.search_text(phrase_query)
513
531
 
532
+ if "sort" not in request.args:
533
+ base_query = base_query.order_by("$text_score")
534
+
514
535
  for filterable in filterables:
515
536
  # If it's from an `nested_filter`, use the custom label instead of the key,
516
537
  # eg use `organization_badge` instead of `organization.badges` which is
@@ -652,6 +673,9 @@ def patch(obj, request) -> type:
652
673
  model_attribute = getattr(obj.__class__, key)
653
674
  info = getattr(model_attribute, "__additional_field_info__", {})
654
675
 
676
+ if value == "" and isinstance(model_attribute, mongo_fields.StringField):
677
+ value = None
678
+
655
679
  if hasattr(model_attribute, "from_input"):
656
680
  value = model_attribute.from_input(value)
657
681
  elif isinstance(model_attribute, mongoengine.fields.ListField) and isinstance(
@@ -677,6 +701,13 @@ def patch(obj, request) -> type:
677
701
  value["id"],
678
702
  document_type=db.resolve_model(value["class"]),
679
703
  )
704
+ elif value and isinstance(
705
+ model_attribute,
706
+ mongoengine.fields.GenericEmbeddedDocumentField,
707
+ ):
708
+ generic_key = info.get("generic_key", DEFAULT_GENERIC_KEY)
709
+ embedded_field = classes_by_names[value[generic_key]]
710
+ value = patch(embedded_field(), value)
680
711
  elif value and isinstance(
681
712
  model_attribute,
682
713
  mongoengine.fields.EmbeddedDocumentField,
udata/app.py CHANGED
@@ -1,26 +1,28 @@
1
- import datetime
2
1
  import logging
3
2
  import os
4
3
  import types
4
+ from datetime import datetime
5
5
  from importlib.metadata import entry_points
6
6
  from os.path import abspath, dirname, exists, isfile, join
7
7
 
8
8
  import bson
9
+ from bson import json_util
9
10
  from flask import Blueprint as BaseBlueprint
10
11
  from flask import (
11
12
  Flask,
12
13
  abort,
13
14
  g,
14
- json,
15
15
  jsonify,
16
16
  make_response,
17
17
  render_template,
18
18
  request,
19
19
  send_from_directory,
20
20
  )
21
+ from flask.json.provider import DefaultJSONProvider
21
22
  from flask_caching import Cache
22
23
  from flask_wtf.csrf import CSRFProtect
23
- from speaklater import is_lazy_string
24
+ from mongoengine import EmbeddedDocument
25
+ from mongoengine.base import BaseDocument
24
26
  from werkzeug.exceptions import NotFound
25
27
  from werkzeug.middleware.proxy_fix import ProxyFix
26
28
 
@@ -108,9 +110,9 @@ class Blueprint(BaseBlueprint):
108
110
  return wrapper
109
111
 
110
112
 
111
- class UDataJsonEncoder(json.JSONEncoder):
113
+ class UdataJsonProvider(DefaultJSONProvider):
112
114
  """
113
- A JSONEncoder subclass to encode unsupported types:
115
+ A JSONProvider subclass to encode unsupported types:
114
116
 
115
117
  - ObjectId
116
118
  - datetime
@@ -120,21 +122,16 @@ class UDataJsonEncoder(json.JSONEncoder):
120
122
  Ensure an app context is always present.
121
123
  """
122
124
 
123
- def default(self, obj):
124
- if is_lazy_string(obj):
125
+ @staticmethod
126
+ def default(obj):
127
+ if isinstance(obj, BaseDocument) or isinstance(obj, EmbeddedDocument):
128
+ return json_util._json_convert(obj.to_mongo())
129
+ elif isinstance(obj, bson.ObjectId):
125
130
  return str(obj)
126
- elif isinstance(obj, bson.objectid.ObjectId):
127
- return str(obj)
128
- elif isinstance(obj, datetime.datetime):
131
+ elif isinstance(obj, datetime):
129
132
  return obj.isoformat()
130
- elif hasattr(obj, "to_dict"):
131
- return obj.to_dict()
132
- elif hasattr(obj, "serialize"):
133
- return obj.serialize()
134
- # Serialize Raw data for Document and EmbeddedDocument.
135
- elif hasattr(obj, "_data"):
136
- return obj._data
137
- return super(UDataJsonEncoder, self).default(obj)
133
+
134
+ return super(UdataJsonProvider, UdataJsonProvider).default(obj)
138
135
 
139
136
 
140
137
  # These loggers are very verbose
@@ -166,10 +163,11 @@ def create_app(config="udata.settings.Defaults", override=None, init_logging=ini
166
163
  if override:
167
164
  app.config.from_object(override)
168
165
 
169
- app.json_encoder = UDataJsonEncoder
166
+ app.json_provider_class = UdataJsonProvider
167
+ app.json = app.json_provider_class(app)
170
168
 
171
169
  # `ujson` doesn't support `cls` parameter https://github.com/ultrajson/ultrajson/issues/124
172
- app.config["RESTX_JSON"] = {"cls": UDataJsonEncoder}
170
+ app.config["RESTX_JSON"] = {"default": UdataJsonProvider.default}
173
171
 
174
172
  app.debug = app.config["DEBUG"] and not app.config["TESTING"]
175
173
 
udata/auth/__init__.py CHANGED
@@ -1,4 +1,5 @@
1
1
  import logging
2
+ import typing as t
2
3
 
3
4
  from flask import render_template
4
5
  from flask_principal import Permission as BasePermission
@@ -10,6 +11,7 @@ from flask_security import Security as Security
10
11
  from flask_security import current_user as current_user
11
12
  from flask_security import login_required as login_required
12
13
  from flask_security import login_user as login_user
14
+ from flask_security import mail_util
13
15
 
14
16
  from . import mails
15
17
 
@@ -24,9 +26,6 @@ def render_security_template(template_name_or_list, **kwargs):
24
26
  return render_template(template_name_or_list, **kwargs)
25
27
 
26
28
 
27
- security = Security()
28
-
29
-
30
29
  class Permission(BasePermission):
31
30
  def __init__(self, *needs):
32
31
  """Let administrator bypass all permissions"""
@@ -36,6 +35,24 @@ class Permission(BasePermission):
36
35
  admin_permission = Permission()
37
36
 
38
37
 
38
+ class NoopMailUtil(mail_util.MailUtil):
39
+ def send_mail(
40
+ self,
41
+ template: str,
42
+ subject: str,
43
+ recipient: str,
44
+ sender: t.Union[str, tuple],
45
+ body: str,
46
+ html: t.Optional[str],
47
+ **kwargs: t.Any,
48
+ ) -> None:
49
+ log.debug(f"Sending mail {subject} to {recipient}")
50
+ log.debug(body)
51
+ log.debug(html)
52
+
53
+ return None
54
+
55
+
39
56
  def init_app(app):
40
57
  from udata.models import datastore
41
58
 
@@ -61,19 +78,25 @@ def init_app(app):
61
78
  app.config["CDATA_BASE_URL"] + "?flash=confirm_error",
62
79
  )
63
80
 
64
- security.init_app(
65
- app,
81
+ # Same logic as in our own mail system :DisableMail
82
+ debug = app.config.get("DEBUG", False)
83
+ send_mail = app.config.get("SEND_MAIL", not debug)
84
+ mail_util_cls = mail_util.MailUtil if send_mail else NoopMailUtil
85
+
86
+ security = Security(
66
87
  datastore,
67
88
  register_blueprint=False,
68
89
  render_template=render_security_template,
69
90
  login_form=ExtendedLoginForm,
70
- confirm_register_form=ExtendedRegisterForm,
71
91
  register_form=ExtendedRegisterForm,
72
92
  reset_password_form=ExtendedResetPasswordForm,
73
93
  forgot_password_form=ExtendedForgotPasswordForm,
74
94
  password_util_cls=UdataPasswordUtil,
95
+ mail_util_cls=mail_util_cls,
75
96
  )
76
97
 
98
+ security.init_app(app, datastore, register_blueprint=False)
99
+
77
100
  security_bp = create_security_blueprint(app, app.extensions["security"], "security_blueprint")
78
101
 
79
102
  app.register_blueprint(security_bp)
udata/auth/forms.py CHANGED
@@ -8,7 +8,7 @@ from flask_security.forms import (
8
8
  ForgotPasswordForm,
9
9
  Form,
10
10
  LoginForm,
11
- RegisterForm,
11
+ RegisterFormV2,
12
12
  ResetPasswordForm,
13
13
  )
14
14
 
@@ -31,7 +31,7 @@ class WithCaptcha:
31
31
  return False
32
32
 
33
33
 
34
- class ExtendedRegisterForm(WithCaptcha, RegisterForm):
34
+ class ExtendedRegisterForm(WithCaptcha, RegisterFormV2):
35
35
  first_name = fields.StringField(
36
36
  _("First name"),
37
37
  [
udata/auth/views.py CHANGED
@@ -84,6 +84,11 @@ def confirm_change_email(token):
84
84
  if flash:
85
85
  return redirect(homepage_url(flash=flash, flash_data=flash_data))
86
86
 
87
+ # Check if the new email is already taken by another user
88
+ existing_user = _datastore.find_user(email=new_email)
89
+ if existing_user and existing_user.id != user.id:
90
+ return redirect(homepage_url(flash="change_email_already_taken"))
91
+
87
92
  if user != current_user:
88
93
  logout_user()
89
94
  login_user(user)
@@ -140,10 +145,8 @@ def create_security_blueprint(app, state, import_name):
140
145
  This creates an I18nBlueprint to use as a base.
141
146
  """
142
147
  bp = I18nBlueprint(
143
- state.blueprint_name,
148
+ "security",
144
149
  import_name,
145
- url_prefix=state.url_prefix,
146
- subdomain=state.subdomain,
147
150
  template_folder="templates",
148
151
  )
149
152
 
udata/commands/serve.py CHANGED
@@ -3,7 +3,7 @@ import os
3
3
 
4
4
  import click
5
5
  from flask import current_app
6
- from flask.cli import DispatchingApp, pass_script_info
6
+ from flask.cli import pass_script_info
7
7
  from werkzeug.serving import run_simple
8
8
 
9
9
  from udata.commands import cli
@@ -26,17 +26,11 @@ log = logging.getLogger(__name__)
26
26
  default=None,
27
27
  help="Enable or disable the debugger. By default the debugger is active if debug is enabled.",
28
28
  )
29
- @click.option(
30
- "--eager-loading/--lazy-loader",
31
- default=None,
32
- help="Enable or disable eager loading. By default eager "
33
- "loading is enabled if the reloader is disabled.",
34
- )
35
29
  @click.option(
36
30
  "--with-threads/--without-threads", default=True, help="Enable or disable multithreading."
37
31
  )
38
32
  @pass_script_info
39
- def serve(info, host, port, reload, debugger, eager_loading, with_threads):
33
+ def serve(info, host, port, reload, debugger, with_threads):
40
34
  """
41
35
  Runs a local udata development server.
42
36
 
@@ -62,10 +56,8 @@ def serve(info, host, port, reload, debugger, eager_loading, with_threads):
62
56
  reload = bool(debug)
63
57
  if debugger is None:
64
58
  debugger = bool(debug)
65
- if eager_loading is None:
66
- eager_loading = not reload
67
59
 
68
- app = DispatchingApp(info.load_app, use_eager_loading=eager_loading)
60
+ app = info.load_app()
69
61
 
70
62
  settings = os.environ.get("UDATA_SETTINGS", os.path.join(os.getcwd(), "udata.cfg"))
71
63
  extra_files = [settings]
@@ -24,7 +24,7 @@ from udata.tests.api import PytestOnlyAPITestCase
24
24
 
25
25
  class FixturesTest(PytestOnlyAPITestCase):
26
26
  @pytest.mark.options(FIXTURE_DATASET_SLUGS=["some-test-dataset-slug"])
27
- def test_generate_fixtures_file_then_import(self, app, cli, api, monkeypatch):
27
+ def test_generate_fixtures_file_then_import(self, mocker):
28
28
  """Test generating fixtures from the current env, then importing them back."""
29
29
  assert models.Dataset.objects.count() == 0 # Start with a clean slate.
30
30
  user = UserFactory()
@@ -55,11 +55,11 @@ class FixturesTest(PytestOnlyAPITestCase):
55
55
  DataserviceFactory(datasets=[dataset], organization=org, contact_points=[contact_point])
56
56
 
57
57
  with NamedTemporaryFile(mode="w+", delete=True) as fixtures_fd:
58
- # Get the fixtures from the local instance.
59
- monkeypatch.setattr(requests, "get", lambda url: api.get(url))
60
- monkeypatch.setattr(Response, "json", Response.get_json)
61
- Response.ok = True
62
- result = cli("generate-fixtures-file", "", fixtures_fd.name)
58
+ # Get the fixtures from the local instance by redirecting requests.get to the test client
59
+ mocker.patch.object(requests, "get", side_effect=lambda url: self.get(url))
60
+ mocker.patch.object(Response, "json", Response.get_json)
61
+ mocker.patch.object(Response, "ok", True, create=True)
62
+ result = self.cli("generate-fixtures-file", "", fixtures_fd.name)
63
63
  fixtures_fd.flush()
64
64
  assert "Fixtures saved to file " in result.output
65
65
 
@@ -81,7 +81,7 @@ class FixturesTest(PytestOnlyAPITestCase):
81
81
  assert models.ContactPoint.objects.count() == 0
82
82
 
83
83
  # Then load them in the database to make sure they're correct.
84
- result = cli("import-fixtures", fixtures_fd.name)
84
+ result = self.cli("import-fixtures", fixtures_fd.name)
85
85
  assert models.Organization.objects(slug=org.slug).count() > 0
86
86
  result_org = models.Organization.objects.get(slug=org.slug)
87
87
  assert result_org.members[0].user.id == user.id
@@ -106,11 +106,11 @@ class FixturesTest(PytestOnlyAPITestCase):
106
106
  assert result_dataservice.organization == org
107
107
  assert result_dataservice.contact_points == [contact_point]
108
108
 
109
- def test_import_fixtures_from_default_file(self, cli):
109
+ def test_import_fixtures_from_default_file(self):
110
110
  """Test importing fixtures from udata.commands.fixture.DEFAULT_FIXTURE_FILE."""
111
111
  # Deactivate spam detection when testing import fixtures
112
112
  SpamMixin.detect_spam_enabled = False
113
- cli("import-fixtures")
113
+ self.cli("import-fixtures")
114
114
  SpamMixin.detect_spam_enabled = True
115
115
  assert models.Organization.objects.count() > 0
116
116
  assert models.Dataset.objects.count() > 0
@@ -13,6 +13,6 @@ class ReasonCategoriesAPI(API):
13
13
  def get(self):
14
14
  """List all limitation reason categories"""
15
15
  return [
16
- {"value": category.value, "label": category.label}
16
+ {"value": category.value, "label": category.label, "definition": category.definition}
17
17
  for category in InspireLimitationCategory
18
18
  ]
@@ -63,31 +63,35 @@ class InspireLimitationCategory(StrEnum):
63
63
  match self:
64
64
  case InspireLimitationCategory.PUBLIC_AUTHORITIES:
65
65
  return _(
66
- "The confidentiality of the proceedings of public authorities, where such confidentiality is provided for by law."
66
+ "Public access to datasets and services would adversely affect the confidentiality of the proceedings of public authorities, where such confidentiality is provided for by law."
67
67
  )
68
68
  case InspireLimitationCategory.INTERNATIONAL_RELATIONS:
69
- return _("International relations, public security or national defence.")
69
+ return _(
70
+ "Public access to datasets and services would adversely affect international relations, public security or national defence."
71
+ )
70
72
  case InspireLimitationCategory.COURSE_OF_JUSTICE:
71
73
  return _(
72
- "The course of justice, the ability of any person to receive a fair trial or the ability of a public authority to conduct an enquiry of a criminal or disciplinary nature."
74
+ "Public access to datasets and services would adversely affect the course of justice, the ability of any person to receive a fair trial or the ability of a public authority to conduct an enquiry of a criminal or disciplinary nature."
73
75
  )
74
76
  case InspireLimitationCategory.COMMERCIAL_CONFIDENTIALITY:
75
77
  return _(
76
- "The confidentiality of commercial or industrial information, where such confidentiality is provided for by national or Community law to protect a legitimate economic interest."
78
+ "Public access to datasets and services would adversely affect the confidentiality of commercial or industrial information, where such confidentiality is provided for by national or Community law to protect a legitimate economic interest, including the public interest in maintaining statistical confidentiality and tax secrecy."
77
79
  )
78
80
  case InspireLimitationCategory.INTELLECTUAL_PROPERTY:
79
- return _("Intellectual property rights.")
81
+ return _(
82
+ "Public access to datasets and services would adversely affect intellectual property rights."
83
+ )
80
84
  case InspireLimitationCategory.PERSONAL_DATA:
81
85
  return _(
82
- "The confidentiality of personal data and/or files relating to a natural person where that person has not consented to the disclosure."
86
+ "Public access to datasets and services would adversely affect the confidentiality of personal data and/or files relating to a natural person where that person has not consented to the disclosure of the information to the public, where such confidentiality is provided for by national or Community law."
83
87
  )
84
88
  case InspireLimitationCategory.VOLUNTARY_SUPPLIER:
85
89
  return _(
86
- "The interests or protection of any person who supplied the information requested on a voluntary basis without being under a legal obligation."
90
+ "Public access to datasets and services would adversely affect the interests or protection of any person who supplied the information requested on a voluntary basis without being under, or capable of being put under, a legal obligation to do so, unless that person has consented to the release of the information concerned."
87
91
  )
88
92
  case InspireLimitationCategory.ENVIRONMENTAL_PROTECTION:
89
93
  return _(
90
- "The protection of the environment to which such information relates, such as the location of rare species."
94
+ "Public access to datasets and services would adversely affect the protection of the environment to which such information relates, such as the location of rare species."
91
95
  )
92
96
  case _:
93
97
  assert_never(self)
@@ -4,8 +4,9 @@ from bson import ObjectId
4
4
  from mongoengine.errors import DoesNotExist
5
5
 
6
6
  from udata.api import API, api, fields
7
- from udata.auth import current_user
7
+ from udata.core.dataset.permissions import OwnableReadPermission
8
8
  from udata.core.organization.api_fields import org_ref_fields
9
+ from udata.core.owned import Owned
9
10
  from udata.core.user.api_fields import user_ref_fields
10
11
  from udata.models import Activity, db
11
12
 
@@ -101,7 +102,7 @@ class SiteActivityAPI(API):
101
102
  # Always return a result even not complete
102
103
  # But log the error (ie. visible in sentry, silent for user)
103
104
  # Can happen when someone manually delete an object in DB (ie. without proper purge)
104
- # - Filter out private items (except for sysadmin users)
105
+ # - Filter out items not visible to the current user
105
106
  safe_items = []
106
107
  for item in qs.queryset.items:
107
108
  try:
@@ -109,10 +110,8 @@ class SiteActivityAPI(API):
109
110
  except DoesNotExist as e:
110
111
  log.error(e, exc_info=True)
111
112
  else:
112
- if hasattr(item.related_to, "private") and (
113
- current_user.is_anonymous or not current_user.sysadmin
114
- ):
115
- if item.related_to.private:
113
+ if isinstance(item.related_to, Owned):
114
+ if not OwnableReadPermission(item.related_to).can():
116
115
  continue
117
116
  safe_items.append(item)
118
117
  qs.queryset.items = safe_items
@@ -9,32 +9,32 @@ class BadgeCommandTest(PytestOnlyDBTestCase):
9
9
  def toggle(self, path_or_id, kind):
10
10
  return self.cli("badges", "toggle", path_or_id, kind)
11
11
 
12
- def test_toggle_badge_on(self, cli):
12
+ def test_toggle_badge_on(self):
13
13
  org = OrganizationFactory()
14
14
 
15
- cli("badges", "toggle", str(org.id), PUBLIC_SERVICE)
15
+ self.cli("badges", "toggle", str(org.id), PUBLIC_SERVICE)
16
16
 
17
17
  org.reload()
18
18
  assert org.badges[0].kind == PUBLIC_SERVICE
19
19
 
20
- def test_toggle_badge_off(self, cli):
20
+ def test_toggle_badge_off(self):
21
21
  org = OrganizationFactory()
22
22
  org.add_badge(PUBLIC_SERVICE)
23
23
  org.add_badge(CERTIFIED)
24
24
 
25
- cli("badges", "toggle", str(org.id), PUBLIC_SERVICE)
25
+ self.cli("badges", "toggle", str(org.id), PUBLIC_SERVICE)
26
26
 
27
27
  org.reload()
28
28
  assert org.badges[0].kind == CERTIFIED
29
29
 
30
- def test_toggle_badge_on_from_file(self, cli):
30
+ def test_toggle_badge_on_from_file(self):
31
31
  orgs = [OrganizationFactory() for _ in range(2)]
32
32
 
33
33
  with NamedTemporaryFile(mode="w") as temp:
34
34
  temp.write("\n".join((str(org.id) for org in orgs)))
35
35
  temp.flush()
36
36
 
37
- cli("badges", "toggle", temp.name, PUBLIC_SERVICE)
37
+ self.cli("badges", "toggle", temp.name, PUBLIC_SERVICE)
38
38
 
39
39
  for org in orgs:
40
40
  org.reload()
udata/core/csv.py CHANGED
@@ -5,6 +5,7 @@ from datetime import date, datetime
5
5
  from io import StringIO
6
6
 
7
7
  from flask import Response, stream_with_context
8
+ from mongoengine.queryset import QuerySet
8
9
 
9
10
  from udata.mongo import db
10
11
  from udata.utils import recursive_get
@@ -35,6 +36,10 @@ class Adapter(object):
35
36
  fields = None
36
37
 
37
38
  def __init__(self, queryset):
39
+ # no_cache() to avoid eating up too much RAM when iterating over large querysets.
40
+ # Applied here rather than upstream to preserve custom QuerySet methods (like with_badge).
41
+ if isinstance(queryset, QuerySet):
42
+ queryset = queryset.no_cache()
38
43
  self.queryset = queryset
39
44
  self._fields = None
40
45
 
@@ -309,7 +309,7 @@ class Dataservice(
309
309
 
310
310
  @field(description="Link to the udata web page for this dataservice", show_as_ref=True)
311
311
  def self_web_url(self, **kwargs):
312
- return cdata_url(f"/dataservices/{self._link_id(**kwargs)}/", **kwargs)
312
+ return cdata_url(f"/dataservices/{self._link_id(**kwargs)}", **kwargs)
313
313
 
314
314
  __metrics_keys__ = [
315
315
  "discussions",
@@ -6,6 +6,7 @@ from udata.core.constants import HVD
6
6
  from udata.core.dataservices.models import Dataservice
7
7
  from udata.core.organization.constants import CERTIFIED, PUBLIC_SERVICE
8
8
  from udata.core.organization.models import Organization
9
+ from udata.core.pages.models import Page
9
10
  from udata.core.topic.models import TopicElement
10
11
  from udata.harvest.models import HarvestJob
11
12
  from udata.models import Discussion, Follow, Transfer
@@ -28,6 +29,12 @@ def purge_dataservices(self):
28
29
  Transfer.objects(subject=dataservice).delete()
29
30
  # Remove dataservices references in Topics
30
31
  TopicElement.objects(element=dataservice).update(element=None)
32
+ # Remove dataservices in pages (mongoengine doesn't support updating a field in a generic embed)
33
+ Page._get_collection().update_many(
34
+ {"blocs.dataservices": dataservice.id},
35
+ {"$pull": {"blocs.$[b].dataservices": dataservice.id}},
36
+ array_filters=[{"b.dataservices": dataservice.id}],
37
+ )
31
38
  # Remove dataservice
32
39
  dataservice.delete()
33
40
 
udata/core/dataset/api.py CHANGED
@@ -531,6 +531,8 @@ class ResourcesAPI(API):
531
531
  f"All resources must be reordered, you provided {len(resources)} "
532
532
  f"out of {len(dataset.resources)}",
533
533
  )
534
+ if any(isinstance(r, dict) and "id" not in r for r in resources):
535
+ api.abort(400, "Each resource must have an 'id' field")
534
536
  if set(r["id"] if isinstance(r, dict) else r for r in resources) != set(
535
537
  str(r.id) for r in dataset.resources
536
538
  ):
@@ -730,7 +730,7 @@ class Dataset(
730
730
  }
731
731
 
732
732
  def self_web_url(self, **kwargs):
733
- return cdata_url(f"/datasets/{self._link_id(**kwargs)}/", **kwargs)
733
+ return cdata_url(f"/datasets/{self._link_id(**kwargs)}", **kwargs)
734
734
 
735
735
  def self_api_url(self, **kwargs):
736
736
  return url_for(
@@ -795,7 +795,7 @@ class Dataset(
795
795
  Resources should be fetched when calling this method.
796
796
  """
797
797
  if self.harvest and self.harvest.modified_at:
798
- return self.harvest.modified_at
798
+ return to_naive_datetime(self.harvest.modified_at)
799
799
  if self.resources:
800
800
  return max([res.last_modified for res in self.resources])
801
801
  else:
@@ -1,3 +1,6 @@
1
+ from flask_principal import Permission as BasePermission
2
+ from flask_principal import RoleNeed
3
+
1
4
  from udata.auth import Permission, UserNeed
2
5
  from udata.core.organization.permissions import (
3
6
  OrganizationAdminNeed,
@@ -22,6 +25,34 @@ class OwnablePermission(Permission):
22
25
  super(OwnablePermission, self).__init__(*needs)
23
26
 
24
27
 
28
+ class OwnableReadPermission(BasePermission):
29
+ """Permission to read a private ownable object.
30
+
31
+ Always grants access if the object is not private.
32
+ For private objects, requires owner, org member, or sysadmin.
33
+
34
+ We inherit from BasePermission instead of udata's Permission because
35
+ Permission automatically adds RoleNeed("admin") to all needs. This means
36
+ a permission with no needs would only allow admins. With BasePermission,
37
+ an empty needs set allows everyone (Flask-Principal returns True when
38
+ self.needs is empty).
39
+ """
40
+
41
+ def __init__(self, obj):
42
+ if not getattr(obj, "private", False):
43
+ super().__init__()
44
+ return
45
+
46
+ needs = [RoleNeed("admin")]
47
+ if obj.organization:
48
+ needs.append(OrganizationAdminNeed(obj.organization.id))
49
+ needs.append(OrganizationEditorNeed(obj.organization.id))
50
+ elif obj.owner:
51
+ needs.append(UserNeed(obj.owner.fs_uniquifier))
52
+
53
+ super().__init__(*needs)
54
+
55
+
25
56
  class DatasetEditPermission(OwnablePermission):
26
57
  """Permissions to edit a Dataset"""
27
58