udata 13.0.1.dev12__py3-none-any.whl → 14.4.1.dev7__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of udata might be problematic. Click here for more details.

Files changed (177) hide show
  1. udata/api/__init__.py +2 -8
  2. udata/api_fields.py +35 -4
  3. udata/app.py +30 -50
  4. udata/auth/__init__.py +29 -6
  5. udata/auth/forms.py +8 -6
  6. udata/auth/views.py +6 -3
  7. udata/commands/__init__.py +2 -14
  8. udata/commands/db.py +13 -25
  9. udata/commands/info.py +0 -16
  10. udata/commands/serve.py +3 -11
  11. udata/commands/tests/test_fixtures.py +9 -9
  12. udata/core/access_type/api.py +1 -1
  13. udata/core/access_type/constants.py +12 -8
  14. udata/core/activity/api.py +5 -6
  15. udata/core/avatars/api.py +43 -0
  16. udata/core/avatars/test_avatar_api.py +30 -0
  17. udata/core/badges/tests/test_commands.py +6 -6
  18. udata/core/csv.py +5 -0
  19. udata/core/dataservices/models.py +15 -3
  20. udata/core/dataservices/tasks.py +7 -0
  21. udata/core/dataset/api.py +2 -0
  22. udata/core/dataset/models.py +2 -2
  23. udata/core/dataset/permissions.py +31 -0
  24. udata/core/dataset/tasks.py +50 -10
  25. udata/core/discussions/models.py +1 -0
  26. udata/core/metrics/__init__.py +0 -6
  27. udata/core/organization/api.py +8 -5
  28. udata/core/organization/mails.py +1 -1
  29. udata/core/organization/models.py +9 -1
  30. udata/core/organization/notifications.py +84 -0
  31. udata/core/organization/permissions.py +1 -1
  32. udata/core/organization/tasks.py +3 -0
  33. udata/core/pages/tests/test_api.py +32 -0
  34. udata/core/post/api.py +24 -69
  35. udata/core/post/models.py +84 -16
  36. udata/core/post/tests/test_api.py +24 -1
  37. udata/core/reports/api.py +18 -0
  38. udata/core/reports/models.py +42 -2
  39. udata/core/reuse/models.py +1 -1
  40. udata/core/reuse/tasks.py +7 -0
  41. udata/core/site/models.py +2 -6
  42. udata/core/spatial/commands.py +2 -4
  43. udata/core/spatial/forms.py +2 -2
  44. udata/core/spatial/models.py +0 -10
  45. udata/core/spatial/tests/test_api.py +1 -36
  46. udata/core/user/models.py +15 -2
  47. udata/cors.py +2 -5
  48. udata/db/migrations.py +279 -0
  49. udata/features/notifications/api.py +7 -18
  50. udata/features/notifications/models.py +56 -0
  51. udata/features/notifications/tasks.py +25 -0
  52. udata/flask_mongoengine/engine.py +0 -4
  53. udata/frontend/__init__.py +3 -122
  54. udata/frontend/markdown.py +2 -1
  55. udata/harvest/actions.py +24 -9
  56. udata/harvest/api.py +30 -22
  57. udata/harvest/backends/__init__.py +21 -9
  58. udata/harvest/backends/base.py +29 -3
  59. udata/harvest/backends/ckan/harvesters.py +13 -2
  60. udata/harvest/backends/dcat.py +3 -0
  61. udata/harvest/backends/maaf.py +1 -0
  62. udata/harvest/commands.py +39 -4
  63. udata/harvest/filters.py +17 -6
  64. udata/harvest/forms.py +9 -6
  65. udata/harvest/models.py +16 -0
  66. udata/harvest/permissions.py +27 -0
  67. udata/harvest/tasks.py +3 -5
  68. udata/harvest/tests/ckan/test_ckan_backend.py +35 -2
  69. udata/harvest/tests/ckan/test_ckan_backend_errors.py +1 -1
  70. udata/harvest/tests/ckan/test_ckan_backend_filters.py +1 -1
  71. udata/harvest/tests/ckan/test_dkan_backend.py +1 -1
  72. udata/harvest/tests/dcat/udata.xml +6 -6
  73. udata/harvest/tests/factories.py +1 -1
  74. udata/harvest/tests/test_actions.py +63 -8
  75. udata/harvest/tests/test_api.py +278 -123
  76. udata/harvest/tests/test_base_backend.py +88 -1
  77. udata/harvest/tests/test_dcat_backend.py +60 -13
  78. udata/harvest/tests/test_filters.py +6 -0
  79. udata/i18n.py +11 -273
  80. udata/mail.py +5 -1
  81. udata/migrations/2025-10-31-create-membership-request-notifications.py +55 -0
  82. udata/migrations/2025-11-13-delete-user-email-index.py +25 -0
  83. udata/migrations/2025-12-04-add-uuid-to-discussion-messages.py +28 -0
  84. udata/models/__init__.py +0 -8
  85. udata/mongo/slug_fields.py +1 -1
  86. udata/rdf.py +45 -6
  87. udata/routing.py +2 -10
  88. udata/sentry.py +4 -10
  89. udata/settings.py +23 -17
  90. udata/tasks.py +4 -3
  91. udata/templates/mail/message.html +5 -31
  92. udata/tests/__init__.py +28 -12
  93. udata/tests/api/__init__.py +108 -21
  94. udata/tests/api/test_activities_api.py +36 -0
  95. udata/tests/api/test_auth_api.py +121 -95
  96. udata/tests/api/test_base_api.py +7 -4
  97. udata/tests/api/test_dataservices_api.py +29 -1
  98. udata/tests/api/test_datasets_api.py +45 -21
  99. udata/tests/api/test_organizations_api.py +192 -197
  100. udata/tests/api/test_reports_api.py +157 -0
  101. udata/tests/api/test_reuses_api.py +147 -147
  102. udata/tests/api/test_security_api.py +12 -12
  103. udata/tests/api/test_swagger.py +4 -4
  104. udata/tests/api/test_tags_api.py +8 -8
  105. udata/tests/api/test_user_api.py +13 -1
  106. udata/tests/apiv2/test_swagger.py +4 -4
  107. udata/tests/apiv2/test_topics.py +1 -1
  108. udata/tests/cli/test_cli_base.py +8 -9
  109. udata/tests/dataset/test_dataset_commands.py +4 -4
  110. udata/tests/dataset/test_dataset_model.py +66 -26
  111. udata/tests/dataset/test_dataset_rdf.py +99 -5
  112. udata/tests/dataset/test_resource_preview.py +0 -1
  113. udata/tests/frontend/test_auth.py +24 -1
  114. udata/tests/frontend/test_csv.py +0 -3
  115. udata/tests/helpers.py +37 -27
  116. udata/tests/organization/test_notifications.py +67 -2
  117. udata/tests/plugin.py +6 -261
  118. udata/tests/site/test_site_csv_exports.py +22 -10
  119. udata/tests/test_activity.py +9 -9
  120. udata/tests/test_cors.py +1 -1
  121. udata/tests/test_dcat_commands.py +2 -2
  122. udata/tests/test_discussions.py +5 -5
  123. udata/tests/test_migrations.py +181 -481
  124. udata/tests/test_notifications.py +15 -57
  125. udata/tests/test_notifications_task.py +43 -0
  126. udata/tests/test_owned.py +81 -1
  127. udata/tests/test_storages.py +25 -19
  128. udata/tests/test_topics.py +77 -61
  129. udata/tests/test_uris.py +33 -0
  130. udata/tests/workers/test_jobs_commands.py +23 -23
  131. udata/translations/ar/LC_MESSAGES/udata.mo +0 -0
  132. udata/translations/ar/LC_MESSAGES/udata.po +187 -108
  133. udata/translations/de/LC_MESSAGES/udata.mo +0 -0
  134. udata/translations/de/LC_MESSAGES/udata.po +187 -108
  135. udata/translations/es/LC_MESSAGES/udata.mo +0 -0
  136. udata/translations/es/LC_MESSAGES/udata.po +187 -108
  137. udata/translations/fr/LC_MESSAGES/udata.mo +0 -0
  138. udata/translations/fr/LC_MESSAGES/udata.po +188 -109
  139. udata/translations/it/LC_MESSAGES/udata.mo +0 -0
  140. udata/translations/it/LC_MESSAGES/udata.po +187 -108
  141. udata/translations/pt/LC_MESSAGES/udata.mo +0 -0
  142. udata/translations/pt/LC_MESSAGES/udata.po +187 -108
  143. udata/translations/sr/LC_MESSAGES/udata.mo +0 -0
  144. udata/translations/sr/LC_MESSAGES/udata.po +187 -108
  145. udata/translations/udata.pot +215 -106
  146. udata/uris.py +0 -2
  147. udata/utils.py +5 -0
  148. udata-14.4.1.dev7.dist-info/METADATA +109 -0
  149. {udata-13.0.1.dev12.dist-info → udata-14.4.1.dev7.dist-info}/RECORD +153 -166
  150. {udata-13.0.1.dev12.dist-info → udata-14.4.1.dev7.dist-info}/entry_points.txt +3 -5
  151. udata/core/followers/views.py +0 -15
  152. udata/core/post/forms.py +0 -30
  153. udata/entrypoints.py +0 -93
  154. udata/features/identicon/__init__.py +0 -0
  155. udata/features/identicon/api.py +0 -13
  156. udata/features/identicon/backends.py +0 -131
  157. udata/features/identicon/tests/__init__.py +0 -0
  158. udata/features/identicon/tests/test_backends.py +0 -18
  159. udata/features/territories/__init__.py +0 -49
  160. udata/features/territories/api.py +0 -25
  161. udata/features/territories/models.py +0 -51
  162. udata/flask_mongoengine/json.py +0 -38
  163. udata/migrations/__init__.py +0 -367
  164. udata/templates/mail/base.html +0 -105
  165. udata/templates/mail/base.txt +0 -6
  166. udata/templates/mail/button.html +0 -3
  167. udata/templates/mail/layouts/1-column.html +0 -19
  168. udata/templates/mail/layouts/2-columns.html +0 -20
  169. udata/templates/mail/layouts/center-panel.html +0 -16
  170. udata/tests/cli/test_db_cli.py +0 -68
  171. udata/tests/features/territories/__init__.py +0 -20
  172. udata/tests/features/territories/test_territories_api.py +0 -185
  173. udata/tests/frontend/test_hooks.py +0 -149
  174. udata-13.0.1.dev12.dist-info/METADATA +0 -133
  175. {udata-13.0.1.dev12.dist-info → udata-14.4.1.dev7.dist-info}/WHEEL +0 -0
  176. {udata-13.0.1.dev12.dist-info → udata-14.4.1.dev7.dist-info}/licenses/LICENSE +0 -0
  177. {udata-13.0.1.dev12.dist-info → udata-14.4.1.dev7.dist-info}/top_level.txt +0 -0
@@ -5,12 +5,14 @@ import pytest
5
5
  from flask import url_for
6
6
  from pytest_mock import MockerFixture
7
7
 
8
+ from udata.core.dataservices.factories import DataserviceFactory
9
+ from udata.core.dataset.factories import DatasetFactory
8
10
  from udata.core.organization.factories import OrganizationFactory
9
11
  from udata.core.user.factories import AdminFactory, UserFactory
12
+ from udata.harvest.backends import get_enabled_backends
10
13
  from udata.models import Member, PeriodicTask
11
14
  from udata.tests.api import PytestOnlyAPITestCase
12
15
  from udata.tests.helpers import assert200, assert201, assert204, assert400, assert403, assert404
13
- from udata.tests.plugin import ApiClient
14
16
  from udata.utils import faker
15
17
 
16
18
  from .. import actions
@@ -18,20 +20,21 @@ from ..models import (
18
20
  VALIDATION_ACCEPTED,
19
21
  VALIDATION_PENDING,
20
22
  VALIDATION_REFUSED,
23
+ HarvestItem,
21
24
  HarvestSource,
22
25
  HarvestSourceValidation,
23
26
  )
24
- from .factories import HarvestSourceFactory, MockBackendsMixin
27
+ from .factories import HarvestJobFactory, HarvestSourceFactory, MockBackendsMixin
25
28
 
26
29
  log = logging.getLogger(__name__)
27
30
 
28
31
 
29
32
  class HarvestAPITest(MockBackendsMixin, PytestOnlyAPITestCase):
30
- def test_list_backends(self, api):
33
+ def test_list_backends(self):
31
34
  """It should fetch the harvest backends list from the API"""
32
- response = api.get(url_for("api.harvest_backends"))
35
+ response = self.get(url_for("api.harvest_backends"))
33
36
  assert200(response)
34
- assert len(response.json) == len(actions.list_backends())
37
+ assert len(response.json) == len(get_enabled_backends())
35
38
  for data in response.json:
36
39
  assert "id" in data
37
40
  assert "label" in data
@@ -39,87 +42,87 @@ class HarvestAPITest(MockBackendsMixin, PytestOnlyAPITestCase):
39
42
  assert isinstance(data["filters"], (list, tuple))
40
43
  assert "extra_configs" in data
41
44
 
42
- def test_list_sources(self, api):
45
+ def test_list_sources(self):
43
46
  sources = HarvestSourceFactory.create_batch(3)
44
47
 
45
- response = api.get(url_for("api.harvest_sources"))
48
+ response = self.get(url_for("api.harvest_sources"))
46
49
  assert200(response)
47
50
  assert len(response.json["data"]) == len(sources)
48
51
 
49
- def test_list_sources_exclude_deleted(self, api):
52
+ def test_list_sources_exclude_deleted(self):
50
53
  sources = HarvestSourceFactory.create_batch(3)
51
54
  HarvestSourceFactory.create_batch(2, deleted=datetime.utcnow())
52
55
 
53
- response = api.get(url_for("api.harvest_sources"))
56
+ response = self.get(url_for("api.harvest_sources"))
54
57
  assert200(response)
55
58
  assert len(response.json["data"]) == len(sources)
56
59
 
57
- def test_list_sources_include_deleted(self, api):
60
+ def test_list_sources_include_deleted(self):
58
61
  sources = HarvestSourceFactory.create_batch(3)
59
62
  sources.extend(HarvestSourceFactory.create_batch(2, deleted=datetime.utcnow()))
60
63
 
61
- response = api.get(url_for("api.harvest_sources", deleted=True))
64
+ response = self.get(url_for("api.harvest_sources", deleted=True))
62
65
  assert200(response)
63
66
  assert len(response.json["data"]) == len(sources)
64
67
 
65
- def test_list_sources_for_owner(self, api):
68
+ def test_list_sources_for_owner(self):
66
69
  owner = UserFactory()
67
70
  sources = HarvestSourceFactory.create_batch(3, owner=owner)
68
71
  HarvestSourceFactory()
69
72
 
70
73
  url = url_for("api.harvest_sources", owner=str(owner.id))
71
- response = api.get(url)
74
+ response = self.get(url)
72
75
  assert200(response)
73
76
 
74
77
  assert len(response.json["data"]) == len(sources)
75
78
 
76
- def test_list_sources_for_org(self, api):
79
+ def test_list_sources_for_org(self):
77
80
  org = OrganizationFactory()
78
81
  sources = HarvestSourceFactory.create_batch(3, organization=org)
79
82
  HarvestSourceFactory()
80
83
 
81
- response = api.get(url_for("api.harvest_sources", owner=str(org.id)))
84
+ response = self.get(url_for("api.harvest_sources", owner=str(org.id)))
82
85
  assert200(response)
83
86
 
84
87
  assert len(response.json["data"]) == len(sources)
85
88
 
86
- def test_list_sources_search(self, api):
89
+ def test_list_sources_search(self):
87
90
  HarvestSourceFactory.create_batch(3)
88
91
  source = HarvestSourceFactory(name="Moissonneur GeoNetwork de la ville de Rennes")
89
92
 
90
93
  url = url_for("api.harvest_sources", q="geonetwork rennes")
91
- response = api.get(url)
94
+ response = self.get(url)
92
95
  assert200(response)
93
96
 
94
97
  assert len(response.json["data"]) == 1
95
98
  assert response.json["data"][0]["id"] == str(source.id)
96
99
 
97
- def test_list_sources_paginate(self, api):
100
+ def test_list_sources_paginate(self):
98
101
  total = 25
99
102
  page_size = 20
100
103
  HarvestSourceFactory.create_batch(total)
101
104
 
102
105
  url = url_for("api.harvest_sources", page=1, page_size=page_size)
103
- response = api.get(url)
106
+ response = self.get(url)
104
107
  assert200(response)
105
108
  assert len(response.json["data"]) == page_size
106
109
  assert response.json["total"] == total
107
110
 
108
111
  url = url_for("api.harvest_sources", page=2, page_size=page_size)
109
- response = api.get(url)
112
+ response = self.get(url)
110
113
  assert200(response)
111
114
  assert len(response.json["data"]) == total - page_size
112
115
  assert response.json["total"] == total
113
116
 
114
117
  url = url_for("api.harvest_sources", page=3, page_size=page_size)
115
- response = api.get(url)
118
+ response = self.get(url)
116
119
  assert404(response)
117
120
 
118
- def test_create_source_with_owner(self, api):
121
+ def test_create_source_with_owner(self):
119
122
  """It should create and attach a new source to an owner"""
120
- user = api.login()
123
+ user = self.login()
121
124
  data = {"name": faker.word(), "url": faker.url(), "backend": "factory"}
122
- response = api.post(url_for("api.harvest_sources"), data)
125
+ response = self.post(url_for("api.harvest_sources"), data)
123
126
 
124
127
  assert201(response)
125
128
 
@@ -128,9 +131,9 @@ class HarvestAPITest(MockBackendsMixin, PytestOnlyAPITestCase):
128
131
  assert source["owner"]["id"] == str(user.id)
129
132
  assert source["organization"] is None
130
133
 
131
- def test_create_source_with_org(self, api):
134
+ def test_create_source_with_org(self):
132
135
  """It should create and attach a new source to an organization"""
133
- user = api.login()
136
+ user = self.login()
134
137
  member = Member(user=user, role="admin")
135
138
  org = OrganizationFactory(members=[member])
136
139
  data = {
@@ -139,7 +142,7 @@ class HarvestAPITest(MockBackendsMixin, PytestOnlyAPITestCase):
139
142
  "backend": "factory",
140
143
  "organization": str(org.id),
141
144
  }
142
- response = api.post(url_for("api.harvest_sources"), data)
145
+ response = self.post(url_for("api.harvest_sources"), data)
143
146
 
144
147
  assert201(response)
145
148
 
@@ -148,9 +151,9 @@ class HarvestAPITest(MockBackendsMixin, PytestOnlyAPITestCase):
148
151
  assert source["owner"] is None
149
152
  assert source["organization"]["id"] == str(org.id)
150
153
 
151
- def test_create_source_with_org_not_member(self, api):
154
+ def test_create_source_with_org_not_member(self):
152
155
  """It should create and attach a new source to an organization"""
153
- user = api.login()
156
+ user = self.login()
154
157
  member = Member(user=user, role="editor")
155
158
  org = OrganizationFactory(members=[member])
156
159
  data = {
@@ -159,13 +162,13 @@ class HarvestAPITest(MockBackendsMixin, PytestOnlyAPITestCase):
159
162
  "backend": "factory",
160
163
  "organization": str(org.id),
161
164
  }
162
- response = api.post(url_for("api.harvest_sources"), data)
165
+ response = self.post(url_for("api.harvest_sources"), data)
163
166
 
164
167
  assert403(response)
165
168
 
166
- def test_create_source_with_config(self, api):
169
+ def test_create_source_with_config(self):
167
170
  """It should create a new source with configuration"""
168
- api.login()
171
+ self.login()
169
172
  data = {
170
173
  "name": faker.word(),
171
174
  "url": faker.url(),
@@ -186,7 +189,7 @@ class HarvestAPITest(MockBackendsMixin, PytestOnlyAPITestCase):
186
189
  ],
187
190
  },
188
191
  }
189
- response = api.post(url_for("api.harvest_sources"), data)
192
+ response = self.post(url_for("api.harvest_sources"), data)
190
193
 
191
194
  assert201(response)
192
195
 
@@ -207,9 +210,9 @@ class HarvestAPITest(MockBackendsMixin, PytestOnlyAPITestCase):
207
210
  ],
208
211
  }
209
212
 
210
- def test_create_source_with_unknown_filter(self, api):
213
+ def test_create_source_with_unknown_filter(self):
211
214
  """Can only use known filters in config"""
212
- api.login()
215
+ self.login()
213
216
  data = {
214
217
  "name": faker.word(),
215
218
  "url": faker.url(),
@@ -220,13 +223,13 @@ class HarvestAPITest(MockBackendsMixin, PytestOnlyAPITestCase):
220
223
  ]
221
224
  },
222
225
  }
223
- response = api.post(url_for("api.harvest_sources"), data)
226
+ response = self.post(url_for("api.harvest_sources"), data)
224
227
 
225
228
  assert400(response)
226
229
 
227
- def test_create_source_with_bad_filter_type(self, api):
230
+ def test_create_source_with_bad_filter_type(self):
228
231
  """Can only use the xpected filter type"""
229
- api.login()
232
+ self.login()
230
233
  data = {
231
234
  "name": faker.word(),
232
235
  "url": faker.url(),
@@ -237,13 +240,13 @@ class HarvestAPITest(MockBackendsMixin, PytestOnlyAPITestCase):
237
240
  ]
238
241
  },
239
242
  }
240
- response = api.post(url_for("api.harvest_sources"), data)
243
+ response = self.post(url_for("api.harvest_sources"), data)
241
244
 
242
245
  assert400(response)
243
246
 
244
- def test_create_source_with_bad_filter_format(self, api):
247
+ def test_create_source_with_bad_filter_format(self):
245
248
  """Filters should have the right format"""
246
- api.login()
249
+ self.login()
247
250
  data = {
248
251
  "name": faker.word(),
249
252
  "url": faker.url(),
@@ -254,13 +257,13 @@ class HarvestAPITest(MockBackendsMixin, PytestOnlyAPITestCase):
254
257
  ]
255
258
  },
256
259
  }
257
- response = api.post(url_for("api.harvest_sources"), data)
260
+ response = self.post(url_for("api.harvest_sources"), data)
258
261
 
259
262
  assert400(response)
260
263
 
261
- def test_create_source_with_unknown_extra_config(self, api):
264
+ def test_create_source_with_unknown_extra_config(self):
262
265
  """Can only use known extra config in config"""
263
- api.login()
266
+ self.login()
264
267
  data = {
265
268
  "name": faker.word(),
266
269
  "url": faker.url(),
@@ -271,13 +274,13 @@ class HarvestAPITest(MockBackendsMixin, PytestOnlyAPITestCase):
271
274
  ]
272
275
  },
273
276
  }
274
- response = api.post(url_for("api.harvest_sources"), data)
277
+ response = self.post(url_for("api.harvest_sources"), data)
275
278
 
276
279
  assert400(response)
277
280
 
278
- def test_create_source_with_bad_extra_config_type(self, api):
281
+ def test_create_source_with_bad_extra_config_type(self):
279
282
  """Can only use the expected extra config type"""
280
- api.login()
283
+ self.login()
281
284
  data = {
282
285
  "name": faker.word(),
283
286
  "url": faker.url(),
@@ -288,13 +291,13 @@ class HarvestAPITest(MockBackendsMixin, PytestOnlyAPITestCase):
288
291
  ]
289
292
  },
290
293
  }
291
- response = api.post(url_for("api.harvest_sources"), data)
294
+ response = self.post(url_for("api.harvest_sources"), data)
292
295
 
293
296
  assert400(response)
294
297
 
295
- def test_create_source_with_bad_extra_config_format(self, api):
298
+ def test_create_source_with_bad_extra_config_format(self):
296
299
  """Extra config should have the right format"""
297
- api.login()
300
+ self.login()
298
301
  data = {
299
302
  "name": faker.word(),
300
303
  "url": faker.url(),
@@ -305,13 +308,13 @@ class HarvestAPITest(MockBackendsMixin, PytestOnlyAPITestCase):
305
308
  ]
306
309
  },
307
310
  }
308
- response = api.post(url_for("api.harvest_sources"), data)
311
+ response = self.post(url_for("api.harvest_sources"), data)
309
312
 
310
313
  assert400(response)
311
314
 
312
- def test_create_source_with_unknown_feature(self, api):
315
+ def test_create_source_with_unknown_feature(self):
313
316
  """Can only use known features in config"""
314
- api.login()
317
+ self.login()
315
318
  data = {
316
319
  "name": faker.word(),
317
320
  "url": faker.url(),
@@ -320,13 +323,13 @@ class HarvestAPITest(MockBackendsMixin, PytestOnlyAPITestCase):
320
323
  "features": {"unknown": True},
321
324
  },
322
325
  }
323
- response = api.post(url_for("api.harvest_sources"), data)
326
+ response = self.post(url_for("api.harvest_sources"), data)
324
327
 
325
328
  assert400(response)
326
329
 
327
- def test_create_source_with_false_feature(self, api):
330
+ def test_create_source_with_false_feature(self):
328
331
  """It should handled negative values"""
329
- api.login()
332
+ self.login()
330
333
  data = {
331
334
  "name": faker.word(),
332
335
  "url": faker.url(),
@@ -338,7 +341,7 @@ class HarvestAPITest(MockBackendsMixin, PytestOnlyAPITestCase):
338
341
  }
339
342
  },
340
343
  }
341
- response = api.post(url_for("api.harvest_sources"), data)
344
+ response = self.post(url_for("api.harvest_sources"), data)
342
345
 
343
346
  assert201(response)
344
347
 
@@ -350,9 +353,9 @@ class HarvestAPITest(MockBackendsMixin, PytestOnlyAPITestCase):
350
353
  }
351
354
  }
352
355
 
353
- def test_create_source_with_not_boolean_feature(self, api):
356
+ def test_create_source_with_not_boolean_feature(self):
354
357
  """It should handled negative values"""
355
- api.login()
358
+ self.login()
356
359
  data = {
357
360
  "name": faker.word(),
358
361
  "url": faker.url(),
@@ -363,28 +366,28 @@ class HarvestAPITest(MockBackendsMixin, PytestOnlyAPITestCase):
363
366
  }
364
367
  },
365
368
  }
366
- response = api.post(url_for("api.harvest_sources"), data)
369
+ response = self.post(url_for("api.harvest_sources"), data)
367
370
 
368
371
  assert400(response)
369
372
 
370
- def test_create_source_with_config_with_custom_key(self, api):
371
- api.login()
373
+ def test_create_source_with_config_with_custom_key(self):
374
+ self.login()
372
375
  data = {
373
376
  "name": faker.word(),
374
377
  "url": faker.url(),
375
378
  "backend": "factory",
376
379
  "config": {"custom": "value"},
377
380
  }
378
- response = api.post(url_for("api.harvest_sources"), data)
381
+ response = self.post(url_for("api.harvest_sources"), data)
379
382
 
380
383
  assert201(response)
381
384
 
382
385
  source = response.json
383
386
  assert source["config"] == {"custom": "value"}
384
387
 
385
- def test_update_source(self, api):
386
- """It should update a source if owner or orga member"""
387
- user = api.login()
388
+ def test_update_source(self):
389
+ """It should update a source if owner or orga admin"""
390
+ user = self.login()
388
391
  source = HarvestSourceFactory(owner=user)
389
392
  new_url = faker.url()
390
393
  data = {
@@ -394,20 +397,20 @@ class HarvestAPITest(MockBackendsMixin, PytestOnlyAPITestCase):
394
397
  "backend": "factory",
395
398
  }
396
399
  api_url = url_for("api.harvest_source", source=source)
397
- response = api.put(api_url, data)
400
+ response = self.put(api_url, data)
398
401
  assert200(response)
399
402
  assert response.json["url"] == new_url
400
403
 
401
- # Source is now owned by orga, with user as member
402
- source.organization = OrganizationFactory(members=[Member(user=user)])
404
+ # Source is now owned by orga, with user as admin
405
+ source.organization = OrganizationFactory(members=[Member(user=user, role="admin")])
403
406
  source.save()
404
407
  api_url = url_for("api.harvest_source", source=source)
405
- response = api.put(api_url, data)
408
+ response = self.put(api_url, data)
406
409
  assert200(response)
407
410
 
408
- def test_update_source_require_permission(self, api):
411
+ def test_update_source_require_permission(self):
409
412
  """It should not update a source if not the owner"""
410
- api.login()
413
+ self.login()
411
414
  source = HarvestSourceFactory()
412
415
  new_url: str = faker.url()
413
416
  data = {
@@ -417,32 +420,32 @@ class HarvestAPITest(MockBackendsMixin, PytestOnlyAPITestCase):
417
420
  "backend": "factory",
418
421
  }
419
422
  api_url: str = url_for("api.harvest_source", source=source)
420
- response = api.put(api_url, data)
423
+ response = self.put(api_url, data)
421
424
 
422
425
  assert403(response)
423
426
 
424
- def test_validate_source(self, api):
427
+ def test_validate_source(self):
425
428
  """It should allow to validate a source if admin"""
426
- user = api.login(AdminFactory())
429
+ user = self.login(AdminFactory())
427
430
  source = HarvestSourceFactory()
428
431
 
429
432
  data = {"state": VALIDATION_ACCEPTED}
430
433
  url = url_for("api.validate_harvest_source", source=source)
431
- response = api.post(url, data)
434
+ response = self.post(url, data)
432
435
  assert200(response)
433
436
 
434
437
  source.reload()
435
438
  assert source.validation.state == VALIDATION_ACCEPTED
436
439
  assert source.validation.by == user
437
440
 
438
- def test_reject_source(self, api):
441
+ def test_reject_source(self):
439
442
  """It should allow to reject a source if admin"""
440
- user = api.login(AdminFactory())
443
+ user = self.login(AdminFactory())
441
444
  source = HarvestSourceFactory()
442
445
 
443
446
  data = {"state": VALIDATION_REFUSED, "comment": "Not valid"}
444
447
  url = url_for("api.validate_harvest_source", source=source)
445
- response = api.post(url, data)
448
+ response = self.post(url, data)
446
449
  assert200(response)
447
450
 
448
451
  source.reload()
@@ -450,40 +453,40 @@ class HarvestAPITest(MockBackendsMixin, PytestOnlyAPITestCase):
450
453
  assert source.validation.comment == "Not valid"
451
454
  assert source.validation.by == user
452
455
 
453
- def test_validate_source_is_admin_only(self, api):
456
+ def test_validate_source_is_admin_only(self):
454
457
  """It should allow to validate a source if admin"""
455
- api.login()
458
+ self.login()
456
459
  source = HarvestSourceFactory()
457
460
 
458
461
  data = {"validate": True}
459
462
  url = url_for("api.validate_harvest_source", source=source)
460
- response = api.post(url, data)
463
+ response = self.post(url, data)
461
464
  assert403(response)
462
465
 
463
- def test_get_source(self, api):
466
+ def test_get_source(self):
464
467
  source = HarvestSourceFactory()
465
468
 
466
469
  url = url_for("api.harvest_source", source=source)
467
- response = api.get(url)
470
+ response = self.get(url)
468
471
  assert200(response)
469
472
 
470
- def test_get_missing_source(self, api):
473
+ def test_get_missing_source(self):
471
474
  url = url_for("api.harvest_source", source="685bb38b9cb9284b93fd9e72")
472
- response = api.get(url)
475
+ response = self.get(url)
473
476
  assert404(response)
474
477
 
475
- def test_source_preview(self, api):
476
- api.login()
477
- source = HarvestSourceFactory(backend="factory")
478
+ def test_source_preview(self):
479
+ user = self.login()
480
+ source = HarvestSourceFactory(backend="factory", owner=user)
478
481
 
479
482
  url = url_for("api.preview_harvest_source", source=source)
480
- response = api.get(url)
483
+ response = self.get(url)
481
484
  assert200(response)
482
485
 
483
486
  @pytest.mark.options(HARVEST_ENABLE_MANUAL_RUN=True)
484
- def test_run_source(self, mocker: MockerFixture, api: ApiClient):
487
+ def test_run_source(self, mocker: MockerFixture):
485
488
  launch = mocker.patch.object(actions.harvest, "delay")
486
- user = api.login()
489
+ user = self.login()
487
490
 
488
491
  source = HarvestSourceFactory(
489
492
  backend="factory",
@@ -492,15 +495,15 @@ class HarvestAPITest(MockBackendsMixin, PytestOnlyAPITestCase):
492
495
  )
493
496
 
494
497
  url = url_for("api.run_harvest_source", source=source)
495
- response = api.post(url)
498
+ response = self.post(url)
496
499
  assert200(response)
497
500
 
498
501
  launch.assert_called()
499
502
 
500
503
  @pytest.mark.options(HARVEST_ENABLE_MANUAL_RUN=False)
501
- def test_cannot_run_source_if_disabled(self, mocker: MockerFixture, api: ApiClient):
504
+ def test_cannot_run_source_if_disabled(self, mocker: MockerFixture):
502
505
  launch = mocker.patch.object(actions.harvest, "delay")
503
- user = api.login()
506
+ user = self.login()
504
507
 
505
508
  source = HarvestSourceFactory(
506
509
  backend="factory",
@@ -509,16 +512,16 @@ class HarvestAPITest(MockBackendsMixin, PytestOnlyAPITestCase):
509
512
  )
510
513
 
511
514
  url = url_for("api.run_harvest_source", source=source)
512
- response = api.post(url)
515
+ response = self.post(url)
513
516
  assert400(response)
514
517
 
515
518
  launch.assert_not_called()
516
519
 
517
520
  @pytest.mark.options(HARVEST_ENABLE_MANUAL_RUN=True)
518
- def test_cannot_run_source_if_not_owned(self, mocker: MockerFixture, api: ApiClient):
521
+ def test_cannot_run_source_if_not_owned(self, mocker: MockerFixture):
519
522
  launch = mocker.patch.object(actions.harvest, "delay")
520
523
  other_user = UserFactory()
521
- api.login()
524
+ self.login()
522
525
 
523
526
  source = HarvestSourceFactory(
524
527
  backend="factory",
@@ -527,15 +530,15 @@ class HarvestAPITest(MockBackendsMixin, PytestOnlyAPITestCase):
527
530
  )
528
531
 
529
532
  url = url_for("api.run_harvest_source", source=source)
530
- response = api.post(url)
533
+ response = self.post(url)
531
534
  assert403(response)
532
535
 
533
536
  launch.assert_not_called()
534
537
 
535
538
  @pytest.mark.options(HARVEST_ENABLE_MANUAL_RUN=True)
536
- def test_cannot_run_source_if_not_validated(self, mocker: MockerFixture, api: ApiClient):
539
+ def test_cannot_run_source_if_not_validated(self, mocker: MockerFixture):
537
540
  launch = mocker.patch.object(actions.harvest, "delay")
538
- user = api.login()
541
+ user = self.login()
539
542
 
540
543
  source = HarvestSourceFactory(
541
544
  backend="factory",
@@ -544,46 +547,46 @@ class HarvestAPITest(MockBackendsMixin, PytestOnlyAPITestCase):
544
547
  )
545
548
 
546
549
  url = url_for("api.run_harvest_source", source=source)
547
- response = api.post(url)
550
+ response = self.post(url)
548
551
  assert400(response)
549
552
 
550
553
  launch.assert_not_called()
551
554
 
552
- def test_source_from_config(self, api):
553
- api.login()
555
+ def test_source_from_config(self):
556
+ self.login()
554
557
  data = {"name": faker.word(), "url": faker.url(), "backend": "factory"}
555
- response = api.post(url_for("api.preview_harvest_source_config"), data)
558
+ response = self.post(url_for("api.preview_harvest_source_config"), data)
556
559
  assert200(response)
557
560
 
558
- def test_delete_source(self, api):
559
- user = api.login()
561
+ def test_delete_source(self):
562
+ user = self.login()
560
563
  source = HarvestSourceFactory(owner=user)
561
564
 
562
565
  url = url_for("api.harvest_source", source=source)
563
- response = api.delete(url)
566
+ response = self.delete(url)
564
567
  assert204(response)
565
568
 
566
569
  deleted_sources = HarvestSource.objects(deleted__exists=True)
567
570
  assert len(deleted_sources) == 1
568
571
 
569
- def test_delete_source_require_permission(self, api):
572
+ def test_delete_source_require_permission(self):
570
573
  """It should not delete a source if not the owner"""
571
- api.login()
574
+ self.login()
572
575
  source = HarvestSourceFactory()
573
576
 
574
577
  url = url_for("api.harvest_source", source=source)
575
- response = api.delete(url)
578
+ response = self.delete(url)
576
579
 
577
580
  assert403(response)
578
581
 
579
- def test_schedule_source(self, api):
582
+ def test_schedule_source(self):
580
583
  """It should allow to schedule a source if admin"""
581
- api.login(AdminFactory())
584
+ self.login(AdminFactory())
582
585
  source = HarvestSourceFactory()
583
586
 
584
587
  data = "0 0 * * *"
585
588
  url = url_for("api.schedule_harvest_source", source=source)
586
- response = api.post(url, data)
589
+ response = self.post(url, data)
587
590
  assert200(response)
588
591
 
589
592
  assert response.json["schedule"] == "0 0 * * *"
@@ -598,22 +601,22 @@ class HarvestAPITest(MockBackendsMixin, PytestOnlyAPITestCase):
598
601
  assert periodic_task.crontab.month_of_year == "*"
599
602
  assert periodic_task.enabled
600
603
 
601
- def test_schedule_source_is_admin_only(self, api):
604
+ def test_schedule_source_is_admin_only(self):
602
605
  """It should only allow admins to schedule a source"""
603
- api.login()
606
+ self.login()
604
607
  source = HarvestSourceFactory()
605
608
 
606
609
  data = "0 0 * * *"
607
610
  url = url_for("api.schedule_harvest_source", source=source)
608
- response = api.post(url, data)
611
+ response = self.post(url, data)
609
612
  assert403(response)
610
613
 
611
614
  source.reload()
612
615
  assert source.periodic_task is None
613
616
 
614
- def test_unschedule_source(self, api):
617
+ def test_unschedule_source(self):
615
618
  """It should allow to unschedule a source if admin"""
616
- api.login(AdminFactory())
619
+ self.login(AdminFactory())
617
620
  periodic_task = PeriodicTask.objects.create(
618
621
  task="harvest",
619
622
  name=faker.name(),
@@ -624,15 +627,15 @@ class HarvestAPITest(MockBackendsMixin, PytestOnlyAPITestCase):
624
627
  source = HarvestSourceFactory(periodic_task=periodic_task)
625
628
 
626
629
  url = url_for("api.schedule_harvest_source", source=source)
627
- response = api.delete(url)
630
+ response = self.delete(url)
628
631
  assert204(response)
629
632
 
630
633
  source.reload()
631
634
  assert source.periodic_task is None
632
635
 
633
- def test_unschedule_source_is_admin_only(self, api):
636
+ def test_unschedule_source_is_admin_only(self):
634
637
  """It should only allow admins to unschedule a source"""
635
- api.login()
638
+ self.login()
636
639
  periodic_task = PeriodicTask.objects.create(
637
640
  task="harvest",
638
641
  name=faker.name(),
@@ -643,8 +646,160 @@ class HarvestAPITest(MockBackendsMixin, PytestOnlyAPITestCase):
643
646
  source = HarvestSourceFactory(periodic_task=periodic_task)
644
647
 
645
648
  url = url_for("api.schedule_harvest_source", source=source)
646
- response = api.delete(url)
649
+ response = self.delete(url)
647
650
  assert403(response)
648
651
 
649
652
  source.reload()
650
653
  assert source.periodic_task is not None
654
+
655
+ def test_list_items(self):
656
+ """It should fetch the harvest items list from the API for a specific job"""
657
+ job = HarvestJobFactory(
658
+ items=[
659
+ HarvestItem(dataset=DatasetFactory()),
660
+ HarvestItem(dataservice=DataserviceFactory()),
661
+ HarvestItem(dataset=DatasetFactory(), remote_url="https://my.remote.example.com"),
662
+ ],
663
+ )
664
+ response = self.get(url_for("api.harvest_job", ident=str(job.id)))
665
+ assert200(response)
666
+ assert len(response.json["items"]) == 3
667
+ assert set(response.json["items"][0].keys()) == set(
668
+ [
669
+ "created",
670
+ "started",
671
+ "ended",
672
+ "dataset",
673
+ "dataservice",
674
+ "remote_url",
675
+ "remote_id",
676
+ "args",
677
+ "errors",
678
+ "kwargs",
679
+ "logs",
680
+ "status",
681
+ ]
682
+ )
683
+ # Make sure appropriate dataset or dataservice is set
684
+ assert response.json["items"][0]["dataset"] is not None
685
+ assert response.json["items"][0]["dataservice"] is None
686
+ assert response.json["items"][1]["dataset"] is None
687
+ assert response.json["items"][1]["dataservice"] is not None
688
+ # Make sure remote_url is exposed if exists
689
+ assert response.json["items"][1]["remote_url"] is None
690
+ assert response.json["items"][2]["remote_url"] == "https://my.remote.example.com"
691
+
692
+ def test_get_source_permissions_as_anonymous(self):
693
+ """It should return all permissions as False for anonymous users"""
694
+ source = HarvestSourceFactory()
695
+
696
+ url = url_for("api.harvest_source", source=source)
697
+ response = self.get(url)
698
+ assert200(response)
699
+
700
+ assert "permissions" in response.json
701
+ permissions = response.json["permissions"]
702
+ assert permissions["edit"] is False
703
+ assert permissions["delete"] is False
704
+ assert permissions["run"] is False
705
+ assert permissions["preview"] is False
706
+ assert permissions["validate"] is False
707
+ assert permissions["schedule"] is False
708
+
709
+ def test_get_source_permissions_as_owner(self):
710
+ """It should return owner permissions as True for source owner"""
711
+ user = self.login()
712
+ source = HarvestSourceFactory(owner=user)
713
+
714
+ url = url_for("api.harvest_source", source=source)
715
+ response = self.get(url)
716
+ assert200(response)
717
+
718
+ permissions = response.json["permissions"]
719
+ assert permissions["edit"] is True
720
+ assert permissions["delete"] is True
721
+ assert permissions["run"] is True
722
+ assert permissions["preview"] is True
723
+ assert permissions["validate"] is False
724
+ assert permissions["schedule"] is False
725
+
726
+ def test_get_source_permissions_as_org_admin(self):
727
+ """It should return owner permissions as True for org admins"""
728
+ user = self.login()
729
+ member = Member(user=user, role="admin")
730
+ org = OrganizationFactory(members=[member])
731
+ source = HarvestSourceFactory(organization=org)
732
+
733
+ url = url_for("api.harvest_source", source=source)
734
+ response = self.get(url)
735
+ assert200(response)
736
+
737
+ permissions = response.json["permissions"]
738
+ assert permissions["edit"] is True
739
+ assert permissions["delete"] is True
740
+ assert permissions["run"] is True
741
+ assert permissions["preview"] is True
742
+ assert permissions["validate"] is False
743
+ assert permissions["schedule"] is False
744
+
745
+ def test_get_source_permissions_as_org_editor(self):
746
+ """It should return only preview permission as True for org editors"""
747
+ user = self.login()
748
+ member = Member(user=user, role="editor")
749
+ org = OrganizationFactory(members=[member])
750
+ source = HarvestSourceFactory(organization=org)
751
+
752
+ url = url_for("api.harvest_source", source=source)
753
+ response = self.get(url)
754
+ assert200(response)
755
+
756
+ permissions = response.json["permissions"]
757
+ assert permissions["edit"] is False
758
+ assert permissions["delete"] is False
759
+ assert permissions["run"] is False
760
+ assert permissions["preview"] is True
761
+ assert permissions["validate"] is False
762
+ assert permissions["schedule"] is False
763
+
764
+ def test_get_source_permissions_as_superadmin(self):
765
+ """It should return all permissions as True for admin users"""
766
+ self.login(AdminFactory())
767
+ source = HarvestSourceFactory()
768
+
769
+ url = url_for("api.harvest_source", source=source)
770
+ response = self.get(url)
771
+ assert200(response)
772
+
773
+ permissions = response.json["permissions"]
774
+ assert permissions["edit"] is True
775
+ assert permissions["delete"] is True
776
+ assert permissions["run"] is True
777
+ assert permissions["preview"] is True
778
+ assert permissions["validate"] is True
779
+ assert permissions["schedule"] is True
780
+
781
+ def test_get_source_permissions_as_other_user(self):
782
+ """It should return all permissions as False for non-owner users"""
783
+ self.login()
784
+ source = HarvestSourceFactory() # owned by another user
785
+
786
+ url = url_for("api.harvest_source", source=source)
787
+ response = self.get(url)
788
+ assert200(response)
789
+
790
+ permissions = response.json["permissions"]
791
+ assert permissions["edit"] is False
792
+ assert permissions["delete"] is False
793
+ assert permissions["run"] is False
794
+ assert permissions["preview"] is False
795
+ assert permissions["validate"] is False
796
+ assert permissions["schedule"] is False
797
+
798
+ def test_preview_source_require_permission(self):
799
+ """It should not allow preview if not the owner"""
800
+ self.login()
801
+ source = HarvestSourceFactory() # owned by another user
802
+
803
+ url = url_for("api.preview_harvest_source", source=source)
804
+ response = self.get(url)
805
+ assert403(response)