udata 13.0.1.dev12__py3-none-any.whl → 14.4.1.dev7__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of udata might be problematic. Click here for more details.

Files changed (177) hide show
  1. udata/api/__init__.py +2 -8
  2. udata/api_fields.py +35 -4
  3. udata/app.py +30 -50
  4. udata/auth/__init__.py +29 -6
  5. udata/auth/forms.py +8 -6
  6. udata/auth/views.py +6 -3
  7. udata/commands/__init__.py +2 -14
  8. udata/commands/db.py +13 -25
  9. udata/commands/info.py +0 -16
  10. udata/commands/serve.py +3 -11
  11. udata/commands/tests/test_fixtures.py +9 -9
  12. udata/core/access_type/api.py +1 -1
  13. udata/core/access_type/constants.py +12 -8
  14. udata/core/activity/api.py +5 -6
  15. udata/core/avatars/api.py +43 -0
  16. udata/core/avatars/test_avatar_api.py +30 -0
  17. udata/core/badges/tests/test_commands.py +6 -6
  18. udata/core/csv.py +5 -0
  19. udata/core/dataservices/models.py +15 -3
  20. udata/core/dataservices/tasks.py +7 -0
  21. udata/core/dataset/api.py +2 -0
  22. udata/core/dataset/models.py +2 -2
  23. udata/core/dataset/permissions.py +31 -0
  24. udata/core/dataset/tasks.py +50 -10
  25. udata/core/discussions/models.py +1 -0
  26. udata/core/metrics/__init__.py +0 -6
  27. udata/core/organization/api.py +8 -5
  28. udata/core/organization/mails.py +1 -1
  29. udata/core/organization/models.py +9 -1
  30. udata/core/organization/notifications.py +84 -0
  31. udata/core/organization/permissions.py +1 -1
  32. udata/core/organization/tasks.py +3 -0
  33. udata/core/pages/tests/test_api.py +32 -0
  34. udata/core/post/api.py +24 -69
  35. udata/core/post/models.py +84 -16
  36. udata/core/post/tests/test_api.py +24 -1
  37. udata/core/reports/api.py +18 -0
  38. udata/core/reports/models.py +42 -2
  39. udata/core/reuse/models.py +1 -1
  40. udata/core/reuse/tasks.py +7 -0
  41. udata/core/site/models.py +2 -6
  42. udata/core/spatial/commands.py +2 -4
  43. udata/core/spatial/forms.py +2 -2
  44. udata/core/spatial/models.py +0 -10
  45. udata/core/spatial/tests/test_api.py +1 -36
  46. udata/core/user/models.py +15 -2
  47. udata/cors.py +2 -5
  48. udata/db/migrations.py +279 -0
  49. udata/features/notifications/api.py +7 -18
  50. udata/features/notifications/models.py +56 -0
  51. udata/features/notifications/tasks.py +25 -0
  52. udata/flask_mongoengine/engine.py +0 -4
  53. udata/frontend/__init__.py +3 -122
  54. udata/frontend/markdown.py +2 -1
  55. udata/harvest/actions.py +24 -9
  56. udata/harvest/api.py +30 -22
  57. udata/harvest/backends/__init__.py +21 -9
  58. udata/harvest/backends/base.py +29 -3
  59. udata/harvest/backends/ckan/harvesters.py +13 -2
  60. udata/harvest/backends/dcat.py +3 -0
  61. udata/harvest/backends/maaf.py +1 -0
  62. udata/harvest/commands.py +39 -4
  63. udata/harvest/filters.py +17 -6
  64. udata/harvest/forms.py +9 -6
  65. udata/harvest/models.py +16 -0
  66. udata/harvest/permissions.py +27 -0
  67. udata/harvest/tasks.py +3 -5
  68. udata/harvest/tests/ckan/test_ckan_backend.py +35 -2
  69. udata/harvest/tests/ckan/test_ckan_backend_errors.py +1 -1
  70. udata/harvest/tests/ckan/test_ckan_backend_filters.py +1 -1
  71. udata/harvest/tests/ckan/test_dkan_backend.py +1 -1
  72. udata/harvest/tests/dcat/udata.xml +6 -6
  73. udata/harvest/tests/factories.py +1 -1
  74. udata/harvest/tests/test_actions.py +63 -8
  75. udata/harvest/tests/test_api.py +278 -123
  76. udata/harvest/tests/test_base_backend.py +88 -1
  77. udata/harvest/tests/test_dcat_backend.py +60 -13
  78. udata/harvest/tests/test_filters.py +6 -0
  79. udata/i18n.py +11 -273
  80. udata/mail.py +5 -1
  81. udata/migrations/2025-10-31-create-membership-request-notifications.py +55 -0
  82. udata/migrations/2025-11-13-delete-user-email-index.py +25 -0
  83. udata/migrations/2025-12-04-add-uuid-to-discussion-messages.py +28 -0
  84. udata/models/__init__.py +0 -8
  85. udata/mongo/slug_fields.py +1 -1
  86. udata/rdf.py +45 -6
  87. udata/routing.py +2 -10
  88. udata/sentry.py +4 -10
  89. udata/settings.py +23 -17
  90. udata/tasks.py +4 -3
  91. udata/templates/mail/message.html +5 -31
  92. udata/tests/__init__.py +28 -12
  93. udata/tests/api/__init__.py +108 -21
  94. udata/tests/api/test_activities_api.py +36 -0
  95. udata/tests/api/test_auth_api.py +121 -95
  96. udata/tests/api/test_base_api.py +7 -4
  97. udata/tests/api/test_dataservices_api.py +29 -1
  98. udata/tests/api/test_datasets_api.py +45 -21
  99. udata/tests/api/test_organizations_api.py +192 -197
  100. udata/tests/api/test_reports_api.py +157 -0
  101. udata/tests/api/test_reuses_api.py +147 -147
  102. udata/tests/api/test_security_api.py +12 -12
  103. udata/tests/api/test_swagger.py +4 -4
  104. udata/tests/api/test_tags_api.py +8 -8
  105. udata/tests/api/test_user_api.py +13 -1
  106. udata/tests/apiv2/test_swagger.py +4 -4
  107. udata/tests/apiv2/test_topics.py +1 -1
  108. udata/tests/cli/test_cli_base.py +8 -9
  109. udata/tests/dataset/test_dataset_commands.py +4 -4
  110. udata/tests/dataset/test_dataset_model.py +66 -26
  111. udata/tests/dataset/test_dataset_rdf.py +99 -5
  112. udata/tests/dataset/test_resource_preview.py +0 -1
  113. udata/tests/frontend/test_auth.py +24 -1
  114. udata/tests/frontend/test_csv.py +0 -3
  115. udata/tests/helpers.py +37 -27
  116. udata/tests/organization/test_notifications.py +67 -2
  117. udata/tests/plugin.py +6 -261
  118. udata/tests/site/test_site_csv_exports.py +22 -10
  119. udata/tests/test_activity.py +9 -9
  120. udata/tests/test_cors.py +1 -1
  121. udata/tests/test_dcat_commands.py +2 -2
  122. udata/tests/test_discussions.py +5 -5
  123. udata/tests/test_migrations.py +181 -481
  124. udata/tests/test_notifications.py +15 -57
  125. udata/tests/test_notifications_task.py +43 -0
  126. udata/tests/test_owned.py +81 -1
  127. udata/tests/test_storages.py +25 -19
  128. udata/tests/test_topics.py +77 -61
  129. udata/tests/test_uris.py +33 -0
  130. udata/tests/workers/test_jobs_commands.py +23 -23
  131. udata/translations/ar/LC_MESSAGES/udata.mo +0 -0
  132. udata/translations/ar/LC_MESSAGES/udata.po +187 -108
  133. udata/translations/de/LC_MESSAGES/udata.mo +0 -0
  134. udata/translations/de/LC_MESSAGES/udata.po +187 -108
  135. udata/translations/es/LC_MESSAGES/udata.mo +0 -0
  136. udata/translations/es/LC_MESSAGES/udata.po +187 -108
  137. udata/translations/fr/LC_MESSAGES/udata.mo +0 -0
  138. udata/translations/fr/LC_MESSAGES/udata.po +188 -109
  139. udata/translations/it/LC_MESSAGES/udata.mo +0 -0
  140. udata/translations/it/LC_MESSAGES/udata.po +187 -108
  141. udata/translations/pt/LC_MESSAGES/udata.mo +0 -0
  142. udata/translations/pt/LC_MESSAGES/udata.po +187 -108
  143. udata/translations/sr/LC_MESSAGES/udata.mo +0 -0
  144. udata/translations/sr/LC_MESSAGES/udata.po +187 -108
  145. udata/translations/udata.pot +215 -106
  146. udata/uris.py +0 -2
  147. udata/utils.py +5 -0
  148. udata-14.4.1.dev7.dist-info/METADATA +109 -0
  149. {udata-13.0.1.dev12.dist-info → udata-14.4.1.dev7.dist-info}/RECORD +153 -166
  150. {udata-13.0.1.dev12.dist-info → udata-14.4.1.dev7.dist-info}/entry_points.txt +3 -5
  151. udata/core/followers/views.py +0 -15
  152. udata/core/post/forms.py +0 -30
  153. udata/entrypoints.py +0 -93
  154. udata/features/identicon/__init__.py +0 -0
  155. udata/features/identicon/api.py +0 -13
  156. udata/features/identicon/backends.py +0 -131
  157. udata/features/identicon/tests/__init__.py +0 -0
  158. udata/features/identicon/tests/test_backends.py +0 -18
  159. udata/features/territories/__init__.py +0 -49
  160. udata/features/territories/api.py +0 -25
  161. udata/features/territories/models.py +0 -51
  162. udata/flask_mongoengine/json.py +0 -38
  163. udata/migrations/__init__.py +0 -367
  164. udata/templates/mail/base.html +0 -105
  165. udata/templates/mail/base.txt +0 -6
  166. udata/templates/mail/button.html +0 -3
  167. udata/templates/mail/layouts/1-column.html +0 -19
  168. udata/templates/mail/layouts/2-columns.html +0 -20
  169. udata/templates/mail/layouts/center-panel.html +0 -16
  170. udata/tests/cli/test_db_cli.py +0 -68
  171. udata/tests/features/territories/__init__.py +0 -20
  172. udata/tests/features/territories/test_territories_api.py +0 -185
  173. udata/tests/frontend/test_hooks.py +0 -149
  174. udata-13.0.1.dev12.dist-info/METADATA +0 -133
  175. {udata-13.0.1.dev12.dist-info → udata-14.4.1.dev7.dist-info}/WHEEL +0 -0
  176. {udata-13.0.1.dev12.dist-info → udata-14.4.1.dev7.dist-info}/licenses/LICENSE +0 -0
  177. {udata-13.0.1.dev12.dist-info → udata-14.4.1.dev7.dist-info}/top_level.txt +0 -0
@@ -24,7 +24,7 @@ from udata.tests.api import PytestOnlyAPITestCase
24
24
 
25
25
  class FixturesTest(PytestOnlyAPITestCase):
26
26
  @pytest.mark.options(FIXTURE_DATASET_SLUGS=["some-test-dataset-slug"])
27
- def test_generate_fixtures_file_then_import(self, app, cli, api, monkeypatch):
27
+ def test_generate_fixtures_file_then_import(self, mocker):
28
28
  """Test generating fixtures from the current env, then importing them back."""
29
29
  assert models.Dataset.objects.count() == 0 # Start with a clean slate.
30
30
  user = UserFactory()
@@ -55,11 +55,11 @@ class FixturesTest(PytestOnlyAPITestCase):
55
55
  DataserviceFactory(datasets=[dataset], organization=org, contact_points=[contact_point])
56
56
 
57
57
  with NamedTemporaryFile(mode="w+", delete=True) as fixtures_fd:
58
- # Get the fixtures from the local instance.
59
- monkeypatch.setattr(requests, "get", lambda url: api.get(url))
60
- monkeypatch.setattr(Response, "json", Response.get_json)
61
- Response.ok = True
62
- result = cli("generate-fixtures-file", "", fixtures_fd.name)
58
+ # Get the fixtures from the local instance by redirecting requests.get to the test client
59
+ mocker.patch.object(requests, "get", side_effect=lambda url: self.get(url))
60
+ mocker.patch.object(Response, "json", Response.get_json)
61
+ mocker.patch.object(Response, "ok", True, create=True)
62
+ result = self.cli("generate-fixtures-file", "", fixtures_fd.name)
63
63
  fixtures_fd.flush()
64
64
  assert "Fixtures saved to file " in result.output
65
65
 
@@ -81,7 +81,7 @@ class FixturesTest(PytestOnlyAPITestCase):
81
81
  assert models.ContactPoint.objects.count() == 0
82
82
 
83
83
  # Then load them in the database to make sure they're correct.
84
- result = cli("import-fixtures", fixtures_fd.name)
84
+ result = self.cli("import-fixtures", fixtures_fd.name)
85
85
  assert models.Organization.objects(slug=org.slug).count() > 0
86
86
  result_org = models.Organization.objects.get(slug=org.slug)
87
87
  assert result_org.members[0].user.id == user.id
@@ -106,11 +106,11 @@ class FixturesTest(PytestOnlyAPITestCase):
106
106
  assert result_dataservice.organization == org
107
107
  assert result_dataservice.contact_points == [contact_point]
108
108
 
109
- def test_import_fixtures_from_default_file(self, cli):
109
+ def test_import_fixtures_from_default_file(self):
110
110
  """Test importing fixtures from udata.commands.fixture.DEFAULT_FIXTURE_FILE."""
111
111
  # Deactivate spam detection when testing import fixtures
112
112
  SpamMixin.detect_spam_enabled = False
113
- cli("import-fixtures")
113
+ self.cli("import-fixtures")
114
114
  SpamMixin.detect_spam_enabled = True
115
115
  assert models.Organization.objects.count() > 0
116
116
  assert models.Dataset.objects.count() > 0
@@ -13,6 +13,6 @@ class ReasonCategoriesAPI(API):
13
13
  def get(self):
14
14
  """List all limitation reason categories"""
15
15
  return [
16
- {"value": category.value, "label": category.label}
16
+ {"value": category.value, "label": category.label, "definition": category.definition}
17
17
  for category in InspireLimitationCategory
18
18
  ]
@@ -63,31 +63,35 @@ class InspireLimitationCategory(StrEnum):
63
63
  match self:
64
64
  case InspireLimitationCategory.PUBLIC_AUTHORITIES:
65
65
  return _(
66
- "The confidentiality of the proceedings of public authorities, where such confidentiality is provided for by law."
66
+ "Public access to datasets and services would adversely affect the confidentiality of the proceedings of public authorities, where such confidentiality is provided for by law."
67
67
  )
68
68
  case InspireLimitationCategory.INTERNATIONAL_RELATIONS:
69
- return _("International relations, public security or national defence.")
69
+ return _(
70
+ "Public access to datasets and services would adversely affect international relations, public security or national defence."
71
+ )
70
72
  case InspireLimitationCategory.COURSE_OF_JUSTICE:
71
73
  return _(
72
- "The course of justice, the ability of any person to receive a fair trial or the ability of a public authority to conduct an enquiry of a criminal or disciplinary nature."
74
+ "Public access to datasets and services would adversely affect the course of justice, the ability of any person to receive a fair trial or the ability of a public authority to conduct an enquiry of a criminal or disciplinary nature."
73
75
  )
74
76
  case InspireLimitationCategory.COMMERCIAL_CONFIDENTIALITY:
75
77
  return _(
76
- "The confidentiality of commercial or industrial information, where such confidentiality is provided for by national or Community law to protect a legitimate economic interest."
78
+ "Public access to datasets and services would adversely affect the confidentiality of commercial or industrial information, where such confidentiality is provided for by national or Community law to protect a legitimate economic interest, including the public interest in maintaining statistical confidentiality and tax secrecy."
77
79
  )
78
80
  case InspireLimitationCategory.INTELLECTUAL_PROPERTY:
79
- return _("Intellectual property rights.")
81
+ return _(
82
+ "Public access to datasets and services would adversely affect intellectual property rights."
83
+ )
80
84
  case InspireLimitationCategory.PERSONAL_DATA:
81
85
  return _(
82
- "The confidentiality of personal data and/or files relating to a natural person where that person has not consented to the disclosure."
86
+ "Public access to datasets and services would adversely affect the confidentiality of personal data and/or files relating to a natural person where that person has not consented to the disclosure of the information to the public, where such confidentiality is provided for by national or Community law."
83
87
  )
84
88
  case InspireLimitationCategory.VOLUNTARY_SUPPLIER:
85
89
  return _(
86
- "The interests or protection of any person who supplied the information requested on a voluntary basis without being under a legal obligation."
90
+ "Public access to datasets and services would adversely affect the interests or protection of any person who supplied the information requested on a voluntary basis without being under, or capable of being put under, a legal obligation to do so, unless that person has consented to the release of the information concerned."
87
91
  )
88
92
  case InspireLimitationCategory.ENVIRONMENTAL_PROTECTION:
89
93
  return _(
90
- "The protection of the environment to which such information relates, such as the location of rare species."
94
+ "Public access to datasets and services would adversely affect the protection of the environment to which such information relates, such as the location of rare species."
91
95
  )
92
96
  case _:
93
97
  assert_never(self)
@@ -4,8 +4,9 @@ from bson import ObjectId
4
4
  from mongoengine.errors import DoesNotExist
5
5
 
6
6
  from udata.api import API, api, fields
7
- from udata.auth import current_user
7
+ from udata.core.dataset.permissions import OwnableReadPermission
8
8
  from udata.core.organization.api_fields import org_ref_fields
9
+ from udata.core.owned import Owned
9
10
  from udata.core.user.api_fields import user_ref_fields
10
11
  from udata.models import Activity, db
11
12
 
@@ -101,7 +102,7 @@ class SiteActivityAPI(API):
101
102
  # Always return a result even not complete
102
103
  # But log the error (ie. visible in sentry, silent for user)
103
104
  # Can happen when someone manually delete an object in DB (ie. without proper purge)
104
- # - Filter out private items (except for sysadmin users)
105
+ # - Filter out items not visible to the current user
105
106
  safe_items = []
106
107
  for item in qs.queryset.items:
107
108
  try:
@@ -109,10 +110,8 @@ class SiteActivityAPI(API):
109
110
  except DoesNotExist as e:
110
111
  log.error(e, exc_info=True)
111
112
  else:
112
- if hasattr(item.related_to, "private") and (
113
- current_user.is_anonymous or not current_user.sysadmin
114
- ):
115
- if item.related_to.private:
113
+ if isinstance(item.related_to, Owned):
114
+ if not OwnableReadPermission(item.related_to).can():
116
115
  continue
117
116
  safe_items.append(item)
118
117
  qs.queryset.items = safe_items
@@ -0,0 +1,43 @@
1
+ import hashlib
2
+ import io
3
+
4
+ import pydenticon
5
+ from flask import current_app, send_file
6
+
7
+ from udata.api import API, api
8
+ from udata.app import cache
9
+
10
+ ns = api.namespace("avatars", "Avatars")
11
+
12
+
13
+ @cache.memoize()
14
+ def generate_pydenticon(identifier, size):
15
+ """
16
+ Use pydenticon to generate an identicon image.
17
+ All parameters are extracted from configuration.
18
+ """
19
+ blocks_size = current_app.config["AVATAR_INTERNAL_SIZE"]
20
+ foreground = current_app.config["AVATAR_INTERNAL_FOREGROUND"]
21
+ background = current_app.config["AVATAR_INTERNAL_BACKGROUND"]
22
+ generator = pydenticon.Generator(
23
+ blocks_size, blocks_size, digest=hashlib.sha1, foreground=foreground, background=background
24
+ )
25
+
26
+ # Pydenticon adds padding to the size and as a consequence
27
+ # we need to compute the size without the padding
28
+ padding = int(round(current_app.config["AVATAR_INTERNAL_PADDING"] * size / 100.0))
29
+ size = size - 2 * padding
30
+ padding = (padding,) * 4
31
+ return generator.generate(identifier, size, size, padding=padding, output_format="png")
32
+
33
+
34
+ @ns.route("/<identifier>/<int:size>/", endpoint="avatar")
35
+ class IdenticonAPI(API):
36
+ @api.doc("avatars")
37
+ def get(self, identifier, size):
38
+ """Get a deterministic avatar given an identifier at a given size"""
39
+ identicon = generate_pydenticon(identifier, size)
40
+ response = send_file(io.BytesIO(identicon), mimetype="image/png")
41
+ etag = hashlib.sha1(identicon).hexdigest()
42
+ response.set_etag(etag)
43
+ return response
@@ -0,0 +1,30 @@
1
+ from flask import url_for
2
+
3
+ from udata.tests.api import PytestOnlyAPITestCase
4
+ from udata.tests.helpers import assert200
5
+ from udata.utils import faker
6
+
7
+
8
+ def assert_stream_equal(response1, response2):
9
+ __tracebackhide__ = True
10
+ stream1 = list(response1.iter_encoded())
11
+ stream2 = list(response2.iter_encoded())
12
+ assert stream1 == stream2
13
+
14
+
15
+ class InternalBackendTest(PytestOnlyAPITestCase):
16
+ def test_base_rendering(self):
17
+ response = self.get(url_for("api.avatar", identifier=faker.word(), size=32))
18
+
19
+ assert200(response)
20
+ assert response.mimetype == "image/png"
21
+ assert response.is_streamed
22
+ etag, weak = response.get_etag()
23
+ assert etag is not None
24
+
25
+ def test_render_twice_the_same(self):
26
+ identifier = faker.word()
27
+ stream_a = self.get(url_for("api.avatar", identifier=identifier, size=32))
28
+ stream_b = self.get(url_for("api.avatar", identifier=identifier, size=32))
29
+
30
+ assert_stream_equal(stream_a, stream_b)
@@ -9,32 +9,32 @@ class BadgeCommandTest(PytestOnlyDBTestCase):
9
9
  def toggle(self, path_or_id, kind):
10
10
  return self.cli("badges", "toggle", path_or_id, kind)
11
11
 
12
- def test_toggle_badge_on(self, cli):
12
+ def test_toggle_badge_on(self):
13
13
  org = OrganizationFactory()
14
14
 
15
- cli("badges", "toggle", str(org.id), PUBLIC_SERVICE)
15
+ self.cli("badges", "toggle", str(org.id), PUBLIC_SERVICE)
16
16
 
17
17
  org.reload()
18
18
  assert org.badges[0].kind == PUBLIC_SERVICE
19
19
 
20
- def test_toggle_badge_off(self, cli):
20
+ def test_toggle_badge_off(self):
21
21
  org = OrganizationFactory()
22
22
  org.add_badge(PUBLIC_SERVICE)
23
23
  org.add_badge(CERTIFIED)
24
24
 
25
- cli("badges", "toggle", str(org.id), PUBLIC_SERVICE)
25
+ self.cli("badges", "toggle", str(org.id), PUBLIC_SERVICE)
26
26
 
27
27
  org.reload()
28
28
  assert org.badges[0].kind == CERTIFIED
29
29
 
30
- def test_toggle_badge_on_from_file(self, cli):
30
+ def test_toggle_badge_on_from_file(self):
31
31
  orgs = [OrganizationFactory() for _ in range(2)]
32
32
 
33
33
  with NamedTemporaryFile(mode="w") as temp:
34
34
  temp.write("\n".join((str(org.id) for org in orgs)))
35
35
  temp.flush()
36
36
 
37
- cli("badges", "toggle", temp.name, PUBLIC_SERVICE)
37
+ self.cli("badges", "toggle", temp.name, PUBLIC_SERVICE)
38
38
 
39
39
  for org in orgs:
40
40
  org.reload()
udata/core/csv.py CHANGED
@@ -5,6 +5,7 @@ from datetime import date, datetime
5
5
  from io import StringIO
6
6
 
7
7
  from flask import Response, stream_with_context
8
+ from mongoengine.queryset import QuerySet
8
9
 
9
10
  from udata.mongo import db
10
11
  from udata.utils import recursive_get
@@ -35,6 +36,10 @@ class Adapter(object):
35
36
  fields = None
36
37
 
37
38
  def __init__(self, queryset):
39
+ # no_cache() to avoid eating up too much RAM when iterating over large querysets.
40
+ # Applied here rather than upstream to preserve custom QuerySet methods (like with_badge).
41
+ if isinstance(queryset, QuerySet):
42
+ queryset = queryset.no_cache()
38
43
  self.queryset = queryset
39
44
  self._fields = None
40
45
 
@@ -130,7 +130,7 @@ def filter_by_topic(base_query, filter_value):
130
130
  try:
131
131
  topic = Topic.objects.get(id=filter_value)
132
132
  except Topic.DoesNotExist:
133
- pass
133
+ return base_query
134
134
  else:
135
135
  return base_query.filter(
136
136
  id__in=[
@@ -140,11 +140,23 @@ def filter_by_topic(base_query, filter_value):
140
140
  )
141
141
 
142
142
 
143
+ def filter_by_reuse(base_query, filter_value):
144
+ from udata.core.reuse.models import Reuse
145
+
146
+ try:
147
+ reuse = Reuse.objects.get(id=filter_value)
148
+ except Reuse.DoesNotExist:
149
+ return base_query
150
+ else:
151
+ return base_query.filter(id__in=[dataservice.id for dataservice in reuse.dataservices])
152
+
153
+
143
154
  @generate_fields(
144
155
  searchable=True,
145
156
  nested_filters={"organization_badge": "organization.badges"},
146
157
  standalone_filters=[
147
- {"key": "topic", "constraints": "objectid", "query": filter_by_topic, "type": str}
158
+ {"key": "topic", "constraints": ["objectid"], "query": filter_by_topic, "type": str},
159
+ {"key": "reuse", "constraints": ["objectid"], "query": filter_by_reuse, "type": str},
148
160
  ],
149
161
  additional_sorts=[
150
162
  {"key": "followers", "value": "metrics.followers"},
@@ -297,7 +309,7 @@ class Dataservice(
297
309
 
298
310
  @field(description="Link to the udata web page for this dataservice", show_as_ref=True)
299
311
  def self_web_url(self, **kwargs):
300
- return cdata_url(f"/dataservices/{self._link_id(**kwargs)}/", **kwargs)
312
+ return cdata_url(f"/dataservices/{self._link_id(**kwargs)}", **kwargs)
301
313
 
302
314
  __metrics_keys__ = [
303
315
  "discussions",
@@ -6,6 +6,7 @@ from udata.core.constants import HVD
6
6
  from udata.core.dataservices.models import Dataservice
7
7
  from udata.core.organization.constants import CERTIFIED, PUBLIC_SERVICE
8
8
  from udata.core.organization.models import Organization
9
+ from udata.core.pages.models import Page
9
10
  from udata.core.topic.models import TopicElement
10
11
  from udata.harvest.models import HarvestJob
11
12
  from udata.models import Discussion, Follow, Transfer
@@ -28,6 +29,12 @@ def purge_dataservices(self):
28
29
  Transfer.objects(subject=dataservice).delete()
29
30
  # Remove dataservices references in Topics
30
31
  TopicElement.objects(element=dataservice).update(element=None)
32
+ # Remove dataservices in pages (mongoengine doesn't support updating a field in a generic embed)
33
+ Page._get_collection().update_many(
34
+ {"blocs.dataservices": dataservice.id},
35
+ {"$pull": {"blocs.$[b].dataservices": dataservice.id}},
36
+ array_filters=[{"b.dataservices": dataservice.id}],
37
+ )
31
38
  # Remove dataservice
32
39
  dataservice.delete()
33
40
 
udata/core/dataset/api.py CHANGED
@@ -531,6 +531,8 @@ class ResourcesAPI(API):
531
531
  f"All resources must be reordered, you provided {len(resources)} "
532
532
  f"out of {len(dataset.resources)}",
533
533
  )
534
+ if any(isinstance(r, dict) and "id" not in r for r in resources):
535
+ api.abort(400, "Each resource must have an 'id' field")
534
536
  if set(r["id"] if isinstance(r, dict) else r for r in resources) != set(
535
537
  str(r.id) for r in dataset.resources
536
538
  ):
@@ -730,7 +730,7 @@ class Dataset(
730
730
  }
731
731
 
732
732
  def self_web_url(self, **kwargs):
733
- return cdata_url(f"/datasets/{self._link_id(**kwargs)}/", **kwargs)
733
+ return cdata_url(f"/datasets/{self._link_id(**kwargs)}", **kwargs)
734
734
 
735
735
  def self_api_url(self, **kwargs):
736
736
  return url_for(
@@ -795,7 +795,7 @@ class Dataset(
795
795
  Resources should be fetched when calling this method.
796
796
  """
797
797
  if self.harvest and self.harvest.modified_at:
798
- return self.harvest.modified_at
798
+ return to_naive_datetime(self.harvest.modified_at)
799
799
  if self.resources:
800
800
  return max([res.last_modified for res in self.resources])
801
801
  else:
@@ -1,3 +1,6 @@
1
+ from flask_principal import Permission as BasePermission
2
+ from flask_principal import RoleNeed
3
+
1
4
  from udata.auth import Permission, UserNeed
2
5
  from udata.core.organization.permissions import (
3
6
  OrganizationAdminNeed,
@@ -22,6 +25,34 @@ class OwnablePermission(Permission):
22
25
  super(OwnablePermission, self).__init__(*needs)
23
26
 
24
27
 
28
+ class OwnableReadPermission(BasePermission):
29
+ """Permission to read a private ownable object.
30
+
31
+ Always grants access if the object is not private.
32
+ For private objects, requires owner, org member, or sysadmin.
33
+
34
+ We inherit from BasePermission instead of udata's Permission because
35
+ Permission automatically adds RoleNeed("admin") to all needs. This means
36
+ a permission with no needs would only allow admins. With BasePermission,
37
+ an empty needs set allows everyone (Flask-Principal returns True when
38
+ self.needs is empty).
39
+ """
40
+
41
+ def __init__(self, obj):
42
+ if not getattr(obj, "private", False):
43
+ super().__init__()
44
+ return
45
+
46
+ needs = [RoleNeed("admin")]
47
+ if obj.organization:
48
+ needs.append(OrganizationAdminNeed(obj.organization.id))
49
+ needs.append(OrganizationEditorNeed(obj.organization.id))
50
+ elif obj.owner:
51
+ needs.append(UserNeed(obj.owner.fs_uniquifier))
52
+
53
+ super().__init__(*needs)
54
+
55
+
25
56
  class DatasetEditPermission(OwnablePermission):
26
57
  """Permissions to edit a Dataset"""
27
58
 
@@ -1,6 +1,7 @@
1
1
  import collections
2
+ import gzip
2
3
  import os
3
- from datetime import datetime
4
+ from datetime import date, datetime
4
5
  from tempfile import NamedTemporaryFile
5
6
 
6
7
  from celery.utils.log import get_task_logger
@@ -15,8 +16,10 @@ from udata.core.dataservices.models import Dataservice
15
16
  from udata.core.dataset.constants import INSPIRE
16
17
  from udata.core.organization.constants import CERTIFIED, PUBLIC_SERVICE
17
18
  from udata.core.organization.models import Organization
19
+ from udata.core.pages.models import Page
18
20
  from udata.harvest.models import HarvestJob
19
21
  from udata.models import Activity, Discussion, Follow, TopicElement, Transfer, db
22
+ from udata.storage.s3 import store_bytes
20
23
  from udata.tasks import job
21
24
 
22
25
  from .models import Checksum, CommunityResource, Dataset, Resource
@@ -53,6 +56,12 @@ def purge_datasets(self):
53
56
  dataservice.update(datasets=datasets)
54
57
  # Remove HarvestItem references
55
58
  HarvestJob.objects(items__dataset=dataset).update(set__items__S__dataset=None)
59
+ # Remove datasets in pages (mongoengine doesn't support updating a field in a generic embed)
60
+ Page._get_collection().update_many(
61
+ {"blocs.datasets": dataset.id},
62
+ {"$pull": {"blocs.$[b].datasets": dataset.id}},
63
+ array_filters=[{"b.datasets": dataset.id}],
64
+ )
56
65
  # Remove associated Transfers
57
66
  Transfer.objects(subject=dataset).delete()
58
67
  # Remove each dataset's resource's file
@@ -86,16 +95,17 @@ def get_queryset(model_cls):
86
95
  for attr in attrs:
87
96
  if getattr(model_cls, attr, None):
88
97
  params[attr] = False
89
- # no_cache to avoid eating up too much RAM
90
- return model_cls.objects.filter(**params).no_cache()
98
+ return model_cls.objects.filter(**params)
99
+
100
+
101
+ def get_resource_for_csv_export_model(model, dataset):
102
+ for resource in dataset.resources:
103
+ if resource.extras.get("csv-export:model", "") == model:
104
+ return resource
91
105
 
92
106
 
93
107
  def get_or_create_resource(r_info, model, dataset):
94
- resource = None
95
- for r in dataset.resources:
96
- if r.extras.get("csv-export:model", "") == model:
97
- resource = r
98
- break
108
+ resource = get_resource_for_csv_export_model(model, dataset)
99
109
  if resource:
100
110
  for k, v in r_info.items():
101
111
  setattr(resource, k, v)
@@ -126,11 +136,16 @@ def store_resource(csvfile, model, dataset):
126
136
  return get_or_create_resource(r_info, model, dataset)
127
137
 
128
138
 
129
- def export_csv_for_model(model, dataset):
139
+ def export_csv_for_model(model, dataset, replace: bool = False):
130
140
  model_cls = getattr(udata_models, model.capitalize(), None)
131
141
  if not model_cls:
132
142
  log.error("Unknow model %s" % model)
133
143
  return
144
+
145
+ fs_filename_to_remove = None
146
+ if existing_resource := get_resource_for_csv_export_model(model, dataset):
147
+ fs_filename_to_remove = existing_resource.fs_filename
148
+
134
149
  queryset = get_queryset(model_cls)
135
150
  adapter = csv.get_adapter(model_cls)
136
151
  if not adapter:
@@ -156,6 +171,15 @@ def export_csv_for_model(model, dataset):
156
171
  else:
157
172
  dataset.last_modified_internal = datetime.utcnow()
158
173
  dataset.save()
174
+ # remove previous catalog if exists and replace is True
175
+ if replace and fs_filename_to_remove:
176
+ try:
177
+ storages.resources.delete(fs_filename_to_remove)
178
+ except FileNotFoundError:
179
+ log.error(
180
+ f"File not found while deleting resource #{resource.id} ({fs_filename_to_remove}) in export_csv_for_model cleanup"
181
+ )
182
+ return resource
159
183
  finally:
160
184
  csvfile.close()
161
185
  os.unlink(csvfile.name)
@@ -184,7 +208,23 @@ def export_csv(self, model=None):
184
208
 
185
209
  models = (model,) if model else ALLOWED_MODELS
186
210
  for model in models:
187
- export_csv_for_model(model, dataset)
211
+ resource = export_csv_for_model(model, dataset, replace=True)
212
+
213
+ # If we are the first day of the month, archive today catalogs
214
+ if (
215
+ current_app.config["EXPORT_CSV_ARCHIVE_S3_BUCKET"]
216
+ and resource
217
+ and date.today().day == 1
218
+ ):
219
+ log.info(
220
+ f"Archiving {model} csv catalog on {current_app.config['EXPORT_CSV_ARCHIVE_S3_BUCKET']} bucket"
221
+ )
222
+ with storages.resources.open(resource.fs_filename, "rb") as f:
223
+ store_bytes(
224
+ bucket=current_app.config["EXPORT_CSV_ARCHIVE_S3_BUCKET"],
225
+ filename=f"{current_app.config['EXPORT_CSV_ARCHIVE_S3_FILENAME_PREFIX']}{resource.title}.gz",
226
+ bytes=gzip.compress(f.read()),
227
+ )
188
228
 
189
229
 
190
230
  @job("bind-tabular-dataservice")
@@ -14,6 +14,7 @@ log = logging.getLogger(__name__)
14
14
 
15
15
 
16
16
  class Message(SpamMixin, db.EmbeddedDocument):
17
+ id = db.AutoUUIDField()
17
18
  content = db.StringField(required=True)
18
19
  posted_on = db.DateTimeField(default=datetime.utcnow, required=True)
19
20
  posted_by = db.ReferenceField("User")
@@ -1,6 +1,3 @@
1
- from udata import entrypoints
2
-
3
-
4
1
  def init_app(app):
5
2
  # Load all core metrics
6
3
  import udata.core.user.metrics # noqa
@@ -9,6 +6,3 @@ def init_app(app):
9
6
  import udata.core.dataset.metrics # noqa
10
7
  import udata.core.reuse.metrics # noqa
11
8
  import udata.core.followers.metrics # noqa
12
-
13
- # Load metrics from plugins
14
- entrypoints.get_enabled("udata.metrics", app)
@@ -383,12 +383,13 @@ class MembershipRequestAPI(API):
383
383
 
384
384
  form = api.validate(MembershipRequestForm, membership_request)
385
385
 
386
- if not membership_request:
386
+ if membership_request:
387
+ form.populate_obj(membership_request)
388
+ org.save()
389
+ else:
387
390
  membership_request = MembershipRequest()
388
- org.requests.append(membership_request)
389
-
390
- form.populate_obj(membership_request)
391
- org.save()
391
+ form.populate_obj(membership_request)
392
+ org.add_membership_request(membership_request)
392
393
 
393
394
  notify_membership_request.delay(str(org.id), str(membership_request.id))
394
395
 
@@ -424,6 +425,7 @@ class MembershipAcceptAPI(MembershipAPI):
424
425
  org.members.append(member)
425
426
  org.count_members()
426
427
  org.save()
428
+ MembershipRequest.after_handle.send(membership_request, org=org)
427
429
 
428
430
  notify_membership_response.delay(str(org.id), str(membership_request.id))
429
431
 
@@ -446,6 +448,7 @@ class MembershipRefuseAPI(MembershipAPI):
446
448
  membership_request.refusal_comment = form.comment.data
447
449
 
448
450
  org.save()
451
+ MembershipRequest.after_handle.send(membership_request, org=org)
449
452
 
450
453
  notify_membership_response.delay(str(org.id), str(membership_request.id))
451
454
 
@@ -16,7 +16,7 @@ def new_membership_request(org: Organization, request: MembershipRequest) -> Mai
16
16
  )
17
17
  ),
18
18
  LabelledContent(_("Reason for the request:"), request.comment),
19
- MailCTA(_("See the request"), cdata_url(f"/admin/organizations/{org.id}/members/")),
19
+ MailCTA(_("See the request"), cdata_url(f"/admin/organizations/{org.id}/members")),
20
20
  ],
21
21
  )
22
22
 
@@ -81,6 +81,9 @@ class MembershipRequest(db.EmbeddedDocument):
81
81
  comment = db.StringField()
82
82
  refusal_comment = db.StringField()
83
83
 
84
+ after_create = Signal()
85
+ after_handle = Signal()
86
+
84
87
  @property
85
88
  def status_label(self):
86
89
  return MEMBERSHIP_STATUS[self.status]
@@ -198,7 +201,7 @@ class Organization(
198
201
  cls.before_save.send(document)
199
202
 
200
203
  def self_web_url(self, **kwargs):
201
- return cdata_url(f"/organizations/{self._link_id(**kwargs)}/", **kwargs)
204
+ return cdata_url(f"/organizations/{self._link_id(**kwargs)}", **kwargs)
202
205
 
203
206
  def self_api_url(self, **kwargs):
204
207
  return url_for(
@@ -304,6 +307,11 @@ class Organization(
304
307
  def views_count(self):
305
308
  return self.metrics.get("views", 0)
306
309
 
310
+ def add_membership_request(self, membership_request):
311
+ self.requests.append(membership_request)
312
+ self.save()
313
+ MembershipRequest.after_create.send(membership_request, org=self)
314
+
307
315
  def count_members(self):
308
316
  self.metrics["members"] = len(self.members)
309
317
  self.save(signal_kwargs={"ignores": ["post_save"]})