udata 12.0.2.dev15__py3-none-any.whl → 13.0.1.dev21__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of udata might be problematic. Click here for more details.

Files changed (258) hide show
  1. udata/api/__init__.py +1 -0
  2. udata/api_fields.py +10 -4
  3. udata/app.py +11 -10
  4. udata/auth/__init__.py +9 -10
  5. udata/auth/mails.py +137 -45
  6. udata/auth/views.py +5 -12
  7. udata/commands/__init__.py +2 -3
  8. udata/commands/info.py +1 -3
  9. udata/commands/tests/test_fixtures.py +6 -3
  10. udata/core/access_type/api.py +18 -0
  11. udata/core/access_type/constants.py +98 -0
  12. udata/core/access_type/models.py +44 -0
  13. udata/core/activity/models.py +1 -1
  14. udata/core/badges/models.py +1 -1
  15. udata/core/badges/tasks.py +35 -1
  16. udata/core/badges/tests/test_commands.py +2 -4
  17. udata/core/badges/tests/test_model.py +2 -2
  18. udata/core/badges/tests/test_tasks.py +55 -0
  19. udata/core/constants.py +1 -0
  20. udata/core/contact_point/models.py +8 -0
  21. udata/core/dataservices/api.py +3 -3
  22. udata/core/dataservices/apiv2.py +3 -1
  23. udata/core/dataservices/constants.py +0 -29
  24. udata/core/dataservices/models.py +44 -44
  25. udata/core/dataservices/rdf.py +2 -1
  26. udata/core/dataservices/search.py +5 -9
  27. udata/core/dataservices/tasks.py +33 -0
  28. udata/core/dataset/api_fields.py +11 -0
  29. udata/core/dataset/apiv2.py +11 -0
  30. udata/core/dataset/constants.py +0 -1
  31. udata/core/dataset/forms.py +29 -0
  32. udata/core/dataset/models.py +16 -4
  33. udata/core/dataset/rdf.py +2 -1
  34. udata/core/dataset/search.py +2 -2
  35. udata/core/dataset/tasks.py +86 -8
  36. udata/core/discussions/mails.py +63 -0
  37. udata/core/discussions/tasks.py +4 -18
  38. udata/core/metrics/__init__.py +0 -6
  39. udata/core/organization/api.py +3 -1
  40. udata/core/organization/mails.py +144 -0
  41. udata/core/organization/models.py +2 -1
  42. udata/core/organization/search.py +1 -1
  43. udata/core/organization/tasks.py +21 -49
  44. udata/core/pages/tests/test_api.py +0 -2
  45. udata/core/reuse/api.py +27 -1
  46. udata/core/reuse/mails.py +21 -0
  47. udata/core/reuse/models.py +10 -1
  48. udata/core/reuse/search.py +1 -1
  49. udata/core/reuse/tasks.py +2 -3
  50. udata/core/site/models.py +2 -6
  51. udata/core/spatial/tests/test_api.py +17 -20
  52. udata/core/spatial/tests/test_models.py +3 -3
  53. udata/core/user/mails.py +54 -0
  54. udata/core/user/models.py +2 -3
  55. udata/core/user/tasks.py +8 -23
  56. udata/core/user/tests/test_user_model.py +2 -6
  57. udata/entrypoints.py +0 -5
  58. udata/features/identicon/tests/test_backends.py +3 -13
  59. udata/forms/fields.py +3 -3
  60. udata/forms/widgets.py +2 -2
  61. udata/frontend/__init__.py +3 -32
  62. udata/harvest/actions.py +4 -9
  63. udata/harvest/api.py +5 -14
  64. udata/harvest/backends/__init__.py +20 -11
  65. udata/harvest/backends/base.py +2 -2
  66. udata/harvest/backends/ckan/harvesters.py +2 -1
  67. udata/harvest/backends/dcat.py +3 -0
  68. udata/harvest/backends/maaf.py +1 -0
  69. udata/harvest/commands.py +6 -4
  70. udata/harvest/forms.py +9 -6
  71. udata/harvest/tasks.py +3 -5
  72. udata/harvest/tests/ckan/test_ckan_backend.py +300 -337
  73. udata/harvest/tests/ckan/test_ckan_backend_errors.py +94 -99
  74. udata/harvest/tests/ckan/test_ckan_backend_filters.py +128 -122
  75. udata/harvest/tests/ckan/test_dkan_backend.py +39 -51
  76. udata/harvest/tests/dcat/datara--5a26b0f6-0ccf-46ad-ac58-734054b91977.rdf.xml +255 -0
  77. udata/harvest/tests/dcat/datara--f40c3860-7236-4b30-a141-23b8ae33f7b2.rdf.xml +289 -0
  78. udata/harvest/tests/factories.py +1 -1
  79. udata/harvest/tests/test_actions.py +11 -9
  80. udata/harvest/tests/test_api.py +4 -5
  81. udata/harvest/tests/test_base_backend.py +5 -4
  82. udata/harvest/tests/test_dcat_backend.py +50 -19
  83. udata/harvest/tests/test_models.py +2 -4
  84. udata/harvest/tests/test_notifications.py +2 -4
  85. udata/harvest/tests/test_tasks.py +2 -3
  86. udata/mail.py +90 -53
  87. udata/migrations/2025-01-05-dataservices-fields-changes.py +8 -14
  88. udata/migrations/2025-10-21-remove-ckan-harvest-modified-at.py +28 -0
  89. udata/migrations/2025-10-29-harvesters-sources-integrity.py +27 -0
  90. udata/mongo/taglist_field.py +3 -3
  91. udata/rdf.py +32 -15
  92. udata/sentry.py +3 -4
  93. udata/settings.py +7 -2
  94. udata/tags.py +5 -5
  95. udata/tasks.py +3 -3
  96. udata/templates/mail/message.html +65 -0
  97. udata/templates/mail/message.txt +16 -0
  98. udata/tests/__init__.py +40 -58
  99. udata/tests/api/__init__.py +87 -2
  100. udata/tests/api/test_activities_api.py +17 -23
  101. udata/tests/api/test_auth_api.py +2 -4
  102. udata/tests/api/test_contact_points.py +48 -54
  103. udata/tests/api/test_dataservices_api.py +57 -37
  104. udata/tests/api/test_datasets_api.py +146 -49
  105. udata/tests/api/test_me_api.py +4 -6
  106. udata/tests/api/test_organizations_api.py +19 -38
  107. udata/tests/api/test_reports_api.py +0 -4
  108. udata/tests/api/test_reuses_api.py +92 -19
  109. udata/tests/api/test_security_api.py +124 -0
  110. udata/tests/api/test_swagger.py +2 -3
  111. udata/tests/api/test_tags_api.py +6 -7
  112. udata/tests/api/test_transfer_api.py +0 -2
  113. udata/tests/api/test_user_api.py +8 -10
  114. udata/tests/apiv2/test_datasets.py +0 -4
  115. udata/tests/apiv2/test_me_api.py +0 -2
  116. udata/tests/apiv2/test_organizations.py +0 -2
  117. udata/tests/apiv2/test_swagger.py +2 -3
  118. udata/tests/apiv2/test_topics.py +0 -2
  119. udata/tests/cli/test_cli_base.py +14 -12
  120. udata/tests/cli/test_db_cli.py +51 -54
  121. udata/tests/contact_point/test_contact_point_models.py +2 -2
  122. udata/tests/dataservice/test_csv_adapter.py +2 -5
  123. udata/tests/dataservice/test_dataservice_rdf.py +8 -6
  124. udata/tests/dataservice/test_dataservice_tasks.py +36 -38
  125. udata/tests/dataset/test_csv_adapter.py +2 -5
  126. udata/tests/dataset/test_dataset_actions.py +2 -4
  127. udata/tests/dataset/test_dataset_commands.py +2 -4
  128. udata/tests/dataset/test_dataset_events.py +3 -3
  129. udata/tests/dataset/test_dataset_model.py +6 -7
  130. udata/tests/dataset/test_dataset_rdf.py +201 -12
  131. udata/tests/dataset/test_dataset_recommendations.py +2 -2
  132. udata/tests/dataset/test_dataset_tasks.py +66 -68
  133. udata/tests/dataset/test_resource_preview.py +39 -48
  134. udata/tests/dataset/test_transport_tasks.py +2 -2
  135. udata/tests/features/territories/__init__.py +0 -6
  136. udata/tests/features/territories/test_territories_api.py +25 -24
  137. udata/tests/forms/test_current_user_field.py +2 -2
  138. udata/tests/forms/test_dict_field.py +2 -4
  139. udata/tests/forms/test_extras_fields.py +2 -3
  140. udata/tests/forms/test_image_field.py +2 -2
  141. udata/tests/forms/test_model_field.py +2 -4
  142. udata/tests/forms/test_publish_as_field.py +2 -4
  143. udata/tests/forms/test_user_forms.py +26 -29
  144. udata/tests/frontend/test_auth.py +2 -3
  145. udata/tests/frontend/test_csv.py +5 -6
  146. udata/tests/frontend/test_error_handlers.py +2 -3
  147. udata/tests/frontend/test_hooks.py +5 -7
  148. udata/tests/frontend/test_markdown.py +3 -4
  149. udata/tests/helpers.py +2 -7
  150. udata/tests/metrics/test_metrics.py +52 -48
  151. udata/tests/metrics/test_tasks.py +154 -150
  152. udata/tests/organization/test_csv_adapter.py +2 -5
  153. udata/tests/organization/test_notifications.py +2 -4
  154. udata/tests/organization/test_organization_model.py +3 -4
  155. udata/tests/organization/test_organization_rdf.py +2 -8
  156. udata/tests/plugin.py +6 -110
  157. udata/tests/reuse/test_reuse_model.py +3 -4
  158. udata/tests/site/test_site_api.py +0 -2
  159. udata/tests/site/test_site_csv_exports.py +0 -2
  160. udata/tests/site/test_site_metrics.py +2 -4
  161. udata/tests/site/test_site_model.py +2 -2
  162. udata/tests/site/test_site_rdf.py +4 -7
  163. udata/tests/test_activity.py +3 -3
  164. udata/tests/test_api_fields.py +6 -9
  165. udata/tests/test_cors.py +0 -2
  166. udata/tests/test_dcat_commands.py +2 -3
  167. udata/tests/test_discussions.py +2 -7
  168. udata/tests/test_mail.py +150 -114
  169. udata/tests/test_migrations.py +413 -419
  170. udata/tests/test_model.py +10 -11
  171. udata/tests/test_notifications.py +2 -3
  172. udata/tests/test_owned.py +3 -3
  173. udata/tests/test_rdf.py +19 -15
  174. udata/tests/test_routing.py +5 -5
  175. udata/tests/test_storages.py +6 -5
  176. udata/tests/test_tags.py +2 -4
  177. udata/tests/test_topics.py +2 -4
  178. udata/tests/test_transfer.py +4 -5
  179. udata/tests/topic/test_topic_tasks.py +25 -27
  180. udata/tests/user/test_user_rdf.py +2 -8
  181. udata/tests/user/test_user_tasks.py +3 -5
  182. udata/tests/workers/test_jobs_commands.py +2 -2
  183. udata/tests/workers/test_tasks_routing.py +27 -27
  184. udata/translations/ar/LC_MESSAGES/udata.mo +0 -0
  185. udata/translations/ar/LC_MESSAGES/udata.po +369 -435
  186. udata/translations/de/LC_MESSAGES/udata.mo +0 -0
  187. udata/translations/de/LC_MESSAGES/udata.po +371 -437
  188. udata/translations/es/LC_MESSAGES/udata.mo +0 -0
  189. udata/translations/es/LC_MESSAGES/udata.po +369 -435
  190. udata/translations/fr/LC_MESSAGES/udata.mo +0 -0
  191. udata/translations/fr/LC_MESSAGES/udata.po +381 -447
  192. udata/translations/it/LC_MESSAGES/udata.mo +0 -0
  193. udata/translations/it/LC_MESSAGES/udata.po +371 -437
  194. udata/translations/pt/LC_MESSAGES/udata.mo +0 -0
  195. udata/translations/pt/LC_MESSAGES/udata.po +371 -437
  196. udata/translations/sr/LC_MESSAGES/udata.mo +0 -0
  197. udata/translations/sr/LC_MESSAGES/udata.po +372 -438
  198. udata/translations/udata.pot +379 -440
  199. udata/utils.py +14 -2
  200. {udata-12.0.2.dev15.dist-info → udata-13.0.1.dev21.dist-info}/METADATA +1 -2
  201. {udata-12.0.2.dev15.dist-info → udata-13.0.1.dev21.dist-info}/RECORD +205 -242
  202. udata/templates/mail/account_deleted.html +0 -5
  203. udata/templates/mail/account_deleted.txt +0 -6
  204. udata/templates/mail/account_inactivity.html +0 -40
  205. udata/templates/mail/account_inactivity.txt +0 -31
  206. udata/templates/mail/badge_added_association.html +0 -33
  207. udata/templates/mail/badge_added_association.txt +0 -11
  208. udata/templates/mail/badge_added_certified.html +0 -33
  209. udata/templates/mail/badge_added_certified.txt +0 -11
  210. udata/templates/mail/badge_added_company.html +0 -33
  211. udata/templates/mail/badge_added_company.txt +0 -11
  212. udata/templates/mail/badge_added_local_authority.html +0 -33
  213. udata/templates/mail/badge_added_local_authority.txt +0 -11
  214. udata/templates/mail/badge_added_public_service.html +0 -33
  215. udata/templates/mail/badge_added_public_service.txt +0 -11
  216. udata/templates/mail/discussion_closed.html +0 -47
  217. udata/templates/mail/discussion_closed.txt +0 -16
  218. udata/templates/mail/inactive_account_deleted.html +0 -5
  219. udata/templates/mail/inactive_account_deleted.txt +0 -6
  220. udata/templates/mail/membership_refused.html +0 -20
  221. udata/templates/mail/membership_refused.txt +0 -11
  222. udata/templates/mail/membership_request.html +0 -46
  223. udata/templates/mail/membership_request.txt +0 -12
  224. udata/templates/mail/new_discussion.html +0 -44
  225. udata/templates/mail/new_discussion.txt +0 -15
  226. udata/templates/mail/new_discussion_comment.html +0 -45
  227. udata/templates/mail/new_discussion_comment.txt +0 -16
  228. udata/templates/mail/new_member.html +0 -27
  229. udata/templates/mail/new_member.txt +0 -11
  230. udata/templates/mail/new_reuse.html +0 -37
  231. udata/templates/mail/new_reuse.txt +0 -9
  232. udata/templates/mail/test.html +0 -6
  233. udata/templates/mail/test.txt +0 -6
  234. udata/templates/mail/user_mail_card.html +0 -26
  235. udata/templates/security/email/base.html +0 -105
  236. udata/templates/security/email/base.txt +0 -6
  237. udata/templates/security/email/button.html +0 -3
  238. udata/templates/security/email/change_notice.html +0 -22
  239. udata/templates/security/email/change_notice.txt +0 -8
  240. udata/templates/security/email/confirmation_instructions.html +0 -20
  241. udata/templates/security/email/confirmation_instructions.txt +0 -7
  242. udata/templates/security/email/login_instructions.html +0 -19
  243. udata/templates/security/email/login_instructions.txt +0 -7
  244. udata/templates/security/email/reset_instructions.html +0 -24
  245. udata/templates/security/email/reset_instructions.txt +0 -9
  246. udata/templates/security/email/reset_notice.html +0 -11
  247. udata/templates/security/email/reset_notice.txt +0 -4
  248. udata/templates/security/email/welcome.html +0 -24
  249. udata/templates/security/email/welcome.txt +0 -9
  250. udata/templates/security/email/welcome_existing.html +0 -32
  251. udata/templates/security/email/welcome_existing.txt +0 -14
  252. udata/terms.md +0 -6
  253. udata/tests/frontend/__init__.py +0 -23
  254. udata/tests/metrics/conftest.py +0 -15
  255. {udata-12.0.2.dev15.dist-info → udata-13.0.1.dev21.dist-info}/WHEEL +0 -0
  256. {udata-12.0.2.dev15.dist-info → udata-13.0.1.dev21.dist-info}/entry_points.txt +0 -0
  257. {udata-12.0.2.dev15.dist-info → udata-13.0.1.dev21.dist-info}/licenses/LICENSE +0 -0
  258. {udata-12.0.2.dev15.dist-info → udata-13.0.1.dev21.dist-info}/top_level.txt +0 -0
@@ -7,7 +7,9 @@ from pytest_mock import MockerFixture
7
7
 
8
8
  from udata.core.organization.factories import OrganizationFactory
9
9
  from udata.core.user.factories import AdminFactory, UserFactory
10
+ from udata.harvest.backends import get_enabled_backends
10
11
  from udata.models import Member, PeriodicTask
12
+ from udata.tests.api import PytestOnlyAPITestCase
11
13
  from udata.tests.helpers import assert200, assert201, assert204, assert400, assert403, assert404
12
14
  from udata.tests.plugin import ApiClient
13
15
  from udata.utils import faker
@@ -25,15 +27,12 @@ from .factories import HarvestSourceFactory, MockBackendsMixin
25
27
  log = logging.getLogger(__name__)
26
28
 
27
29
 
28
- @pytest.mark.usefixtures("clean_db")
29
- class HarvestAPITest(MockBackendsMixin):
30
- modules = []
31
-
30
+ class HarvestAPITest(MockBackendsMixin, PytestOnlyAPITestCase):
32
31
  def test_list_backends(self, api):
33
32
  """It should fetch the harvest backends list from the API"""
34
33
  response = api.get(url_for("api.harvest_backends"))
35
34
  assert200(response)
36
- assert len(response.json) == len(actions.list_backends())
35
+ assert len(response.json) == len(get_enabled_backends())
37
36
  for data in response.json:
38
37
  assert "id" in data
39
38
  assert "label" in data
@@ -10,6 +10,7 @@ from udata.core.dataset import tasks
10
10
  from udata.core.dataset.factories import DatasetFactory
11
11
  from udata.harvest.models import HarvestItem
12
12
  from udata.models import Dataset
13
+ from udata.tests.api import PytestOnlyDBTestCase
13
14
  from udata.tests.helpers import assert_equal_dates
14
15
  from udata.utils import faker
15
16
 
@@ -28,6 +29,8 @@ def gen_remote_IDs(num: int, prefix: str = "") -> list[str]:
28
29
 
29
30
 
30
31
  class FakeBackend(BaseBackend):
32
+ name = "fake-backend"
33
+ display_name = "Fake Backend"
31
34
  filters = (
32
35
  HarvestFilter("First filter", "first", str),
33
36
  HarvestFilter("Second filter", "second", str),
@@ -93,8 +96,7 @@ class HarvestFilterTest:
93
96
  HarvestFilter(faker.word(), faker.word(), type, faker.sentence())
94
97
 
95
98
 
96
- @pytest.mark.usefixtures("clean_db")
97
- class BaseBackendTest:
99
+ class BaseBackendTest(PytestOnlyDBTestCase):
98
100
  def test_simple_harvest(self):
99
101
  now = datetime.utcnow()
100
102
  nb_datasets = 3
@@ -420,8 +422,7 @@ class BaseBackendTest:
420
422
  assert dataset_reused_uri.harvest.source_id == str(source.id)
421
423
 
422
424
 
423
- @pytest.mark.usefixtures("clean_db")
424
- class BaseBackendValidateTest:
425
+ class BaseBackendValidateTest(PytestOnlyDBTestCase):
425
426
  @pytest.fixture
426
427
  def validate(self):
427
428
  return FakeBackend(HarvestSourceFactory()).validate
@@ -18,6 +18,7 @@ from udata.harvest.models import HarvestJob
18
18
  from udata.models import Dataset
19
19
  from udata.rdf import DCAT, RDF, namespace_manager
20
20
  from udata.storage.s3 import get_from_json
21
+ from udata.tests.api import PytestOnlyDBTestCase
21
22
 
22
23
  from .. import actions
23
24
  from ..backends.dcat import URIS_TO_REPLACE
@@ -67,9 +68,8 @@ def mock_csw_pagination(rmock, path, pattern):
67
68
  return url
68
69
 
69
70
 
70
- @pytest.mark.usefixtures("clean_db")
71
- @pytest.mark.options(PLUGINS=["dcat"])
72
- class DcatBackendTest:
71
+ @pytest.mark.options(HARVESTER_BACKENDS=["dcat"])
72
+ class DcatBackendTest(PytestOnlyDBTestCase):
73
73
  def test_simple_flat(self, rmock):
74
74
  filename = "flat.jsonld"
75
75
  url = mock_dcat(rmock, filename)
@@ -191,7 +191,6 @@ class DcatBackendTest:
191
191
 
192
192
  def test_harvest_dataservices_keep_attached_associated_datasets(self, rmock):
193
193
  """It should update the existing list of dataservice.datasets and not overwrite existing ones"""
194
- rmock.get("https://example.com/schemas", json=ResourceSchemaMockData.get_mock_data())
195
194
 
196
195
  filename = "bnodes.xml"
197
196
  url = mock_dcat(rmock, filename)
@@ -359,10 +358,8 @@ class DcatBackendTest:
359
358
  is None
360
359
  )
361
360
 
362
- @pytest.mark.options(SCHEMA_CATALOG_URL="https://example.com/schemas", HARVEST_MAX_ITEMS=2)
361
+ @pytest.mark.options(HARVEST_MAX_ITEMS=2)
363
362
  def test_harvest_max_items(self, rmock):
364
- rmock.get("https://example.com/schemas", json=ResourceSchemaMockData.get_mock_data())
365
-
366
363
  filename = "bnodes.xml"
367
364
  url = mock_dcat(rmock, filename)
368
365
  org = OrganizationFactory()
@@ -373,10 +370,7 @@ class DcatBackendTest:
373
370
  assert Dataset.objects.count() == 2
374
371
  assert HarvestJob.objects.first().status == "done"
375
372
 
376
- @pytest.mark.options(SCHEMA_CATALOG_URL="https://example.com/schemas")
377
373
  def test_harvest_spatial(self, rmock):
378
- rmock.get("https://example.com/schemas", json=ResourceSchemaMockData.get_mock_data())
379
-
380
374
  filename = "bnodes.xml"
381
375
  url = mock_dcat(rmock, filename)
382
376
  org = OrganizationFactory()
@@ -445,10 +439,7 @@ class DcatBackendTest:
445
439
  assert resources_by_title["Resource 3-1"].schema.url is None
446
440
  assert resources_by_title["Resource 3-1"].schema.version == "2.2.0"
447
441
 
448
- @pytest.mark.options(SCHEMA_CATALOG_URL="https://example.com/schemas")
449
442
  def test_harvest_inspire_themese(self, rmock):
450
- rmock.get("https://example.com/schemas", json=ResourceSchemaMockData.get_mock_data())
451
-
452
443
  filename = "bnodes.xml"
453
444
  url = mock_dcat(rmock, filename)
454
445
  org = OrganizationFactory()
@@ -723,6 +714,48 @@ class DcatBackendTest:
723
714
  ) # noqa
724
715
  assert dataset.harvest.last_update.date() == date.today()
725
716
 
717
+ def test_datara_extended_roles_foaf(self, rmock):
718
+ # Converted manually from ISO-19139 using SEMICeu XSLT (tag geodcat-ap-2.0.0)
719
+ url = mock_dcat(rmock, "datara--5a26b0f6-0ccf-46ad-ac58-734054b91977.rdf.xml")
720
+ org = OrganizationFactory()
721
+ source = HarvestSourceFactory(backend="dcat", url=url, organization=org)
722
+ actions.run(source)
723
+ dataset = Dataset.objects.filter(organization=org).first()
724
+
725
+ assert dataset is not None
726
+ assert len(dataset.contact_points) == 2
727
+
728
+ assert dataset.contact_points[0].name == "IGN"
729
+ assert dataset.contact_points[0].email == "sav.bd@ign.fr"
730
+ assert dataset.contact_points[0].role == "rightsHolder"
731
+
732
+ assert dataset.contact_points[1].name == "Administrateur de Données"
733
+ assert dataset.contact_points[1].email == "sig.dreal-ara@developpement-durable.gouv.fr"
734
+ assert dataset.contact_points[1].role == "user"
735
+
736
+ def test_datara_extended_roles_vcard(self, rmock):
737
+ # Converted manually from ISO-19139 using SEMICeu XSLT (tag geodcat-ap-2.0.0)
738
+ url = mock_dcat(rmock, "datara--f40c3860-7236-4b30-a141-23b8ae33f7b2.rdf.xml")
739
+ org = OrganizationFactory()
740
+ source = HarvestSourceFactory(backend="dcat", url=url, organization=org)
741
+ actions.run(source)
742
+ dataset = Dataset.objects.filter(organization=org).first()
743
+
744
+ assert dataset is not None
745
+ assert len(dataset.contact_points) == 3
746
+
747
+ assert dataset.contact_points[0].name == "Administrateur de Données"
748
+ assert dataset.contact_points[0].email == "sig.dreal-ara@developpement-durable.gouv.fr"
749
+ assert dataset.contact_points[0].role == "contact"
750
+
751
+ assert dataset.contact_points[1].name == "Jean-Michel GENIS"
752
+ assert dataset.contact_points[1].email == "jm.genis@cbn-alpin.fr"
753
+ assert dataset.contact_points[1].role == "rightsHolder"
754
+
755
+ assert dataset.contact_points[2].name == "Conservatoire Botanique National Massif Central"
756
+ assert dataset.contact_points[2].email == "Benoit.Renaux@cbnmc.fr"
757
+ assert dataset.contact_points[2].role == "rightsHolder"
758
+
726
759
  def test_udata_xml_catalog(self, rmock):
727
760
  LicenseFactory(id="fr-lo", title="Licence ouverte / Open Licence")
728
761
  url = mock_dcat(rmock, "udata.xml")
@@ -893,9 +926,8 @@ class DcatBackendTest:
893
926
  assert "404 Client Error" in job.errors[0].message
894
927
 
895
928
 
896
- @pytest.mark.usefixtures("clean_db")
897
- @pytest.mark.options(PLUGINS=["csw"])
898
- class CswDcatBackendTest:
929
+ @pytest.mark.options(HARVESTER_BACKENDS=["csw*"])
930
+ class CswDcatBackendTest(PytestOnlyDBTestCase):
899
931
  def test_geonetworkv4(self, rmock):
900
932
  url = mock_csw_pagination(rmock, "geonetwork/srv/eng/csw.rdf", "geonetworkv4-page-{}.xml")
901
933
  org = OrganizationFactory()
@@ -1044,9 +1076,8 @@ class CswDcatBackendTest:
1044
1076
  assert len(job.items) == 1
1045
1077
 
1046
1078
 
1047
- @pytest.mark.usefixtures("clean_db")
1048
- @pytest.mark.options(PLUGINS=["csw"])
1049
- class CswIso19139DcatBackendTest:
1079
+ @pytest.mark.options(HARVESTER_BACKENDS=["csw*"])
1080
+ class CswIso19139DcatBackendTest(PytestOnlyDBTestCase):
1050
1081
  @pytest.mark.parametrize(
1051
1082
  "remote_url_prefix",
1052
1083
  [
@@ -1,7 +1,6 @@
1
1
  import logging
2
2
 
3
- import pytest
4
-
3
+ from udata.tests.api import PytestOnlyDBTestCase
5
4
  from udata.utils import faker
6
5
 
7
6
  from ..models import HarvestSource
@@ -9,8 +8,7 @@ from ..models import HarvestSource
9
8
  log = logging.getLogger(__name__)
10
9
 
11
10
 
12
- @pytest.mark.usefixtures("clean_db")
13
- class HarvestSourceTest:
11
+ class HarvestSourceTest(PytestOnlyDBTestCase):
14
12
  def test_defaults(self):
15
13
  source = HarvestSource.objects.create(name="Test", url=faker.url(), backend="factory")
16
14
  assert source.name == "Test"
@@ -1,14 +1,12 @@
1
- import pytest
2
-
3
1
  from udata.core.user.factories import AdminFactory, UserFactory
4
2
  from udata.harvest.notifications import validate_harvester_notifications
3
+ from udata.tests.api import PytestOnlyDBTestCase
5
4
  from udata.tests.helpers import assert_equal_dates
6
5
 
7
6
  from .factories import HarvestSourceFactory
8
7
 
9
8
 
10
- @pytest.mark.usefixtures("clean_db")
11
- class HarvestNotificationsTest:
9
+ class HarvestNotificationsTest(PytestOnlyDBTestCase):
12
10
  def test_pending_harvester_validations(self):
13
11
  source = HarvestSourceFactory()
14
12
  admin = AdminFactory()
@@ -1,14 +1,13 @@
1
1
  import logging
2
2
 
3
- import pytest
3
+ from udata.tests.api import PytestOnlyDBTestCase
4
4
 
5
5
  from ..tasks import purge_harvest_jobs, purge_harvest_sources
6
6
 
7
7
  log = logging.getLogger(__name__)
8
8
 
9
9
 
10
- @pytest.mark.usefixtures("clean_db")
11
- class HarvestActionsTest:
10
+ class HarvestActionsTest(PytestOnlyDBTestCase):
12
11
  def test_purge_sources(self, mocker):
13
12
  """It should purge from DB sources flagged as deleted"""
14
13
  mock = mocker.patch("udata.harvest.actions.purge_sources")
udata/mail.py CHANGED
@@ -1,9 +1,11 @@
1
+ import copy
1
2
  import logging
2
- from contextlib import contextmanager
3
- from smtplib import SMTPException
3
+ from dataclasses import dataclass
4
+ from html import escape
4
5
 
5
6
  from blinker import signal
6
7
  from flask import current_app, render_template
8
+ from flask_babel import LazyString
7
9
  from flask_mail import Mail, Message
8
10
 
9
11
  from udata import i18n
@@ -15,69 +17,104 @@ mail = Mail()
15
17
  mail_sent = signal("mail-sent")
16
18
 
17
19
 
18
- class FakeMailer(object):
19
- """Display sent mail in logging output"""
20
+ @dataclass
21
+ class MailCTA:
22
+ label: LazyString
23
+ link: str | None
20
24
 
21
- def send(self, msg):
22
- log.debug(msg.body)
23
- log.debug(msg.html)
24
- mail_sent.send(msg)
25
25
 
26
+ @dataclass
27
+ class LabelledContent:
28
+ label: LazyString
29
+ content: str
30
+ inline: bool = False
31
+ truncated_at: int = 50
26
32
 
27
- @contextmanager
28
- def dummyconnection(*args, **kw):
29
- """Allow to test email templates rendering without actually send emails."""
30
- yield FakeMailer()
33
+ @property
34
+ def truncated_content(self) -> str:
35
+ return (
36
+ self.content[: self.truncated_at] + "…"
37
+ if len(self.content) > self.truncated_at
38
+ else self.content
39
+ )
31
40
 
32
41
 
33
- def init_app(app):
34
- mail.init_app(app)
42
+ @dataclass
43
+ class ParagraphWithLinks:
44
+ paragraph: LazyString
35
45
 
46
+ def __str__(self):
47
+ return str(self.paragraph)
36
48
 
37
- def send(subject, recipients, template_base, **kwargs):
38
- """
39
- Send a given email to multiple recipients.
49
+ @property
50
+ def html(self):
51
+ new_paragraph = copy.deepcopy(self.paragraph)
40
52
 
41
- User prefered language is taken in account.
42
- To translate the subject in the right language, you should ugettext_lazy
43
- """
44
- sender = kwargs.pop("sender", None)
45
- if not isinstance(recipients, (list, tuple)):
46
- recipients = [recipients]
53
+ for key, value in new_paragraph._kwargs.items():
54
+ if hasattr(value, "url_for"):
55
+ new_paragraph._kwargs[key] = (
56
+ f'<a href="{value.url_for(_mailCampaign=True)}" style="color: #000000; text-decoration: underline;">{escape(str(value))}</a>'
57
+ )
58
+
59
+ return str(new_paragraph)
60
+
61
+
62
+ @dataclass
63
+ class MailMessage:
64
+ subject: LazyString
65
+ paragraphs: list[LazyString | MailCTA | ParagraphWithLinks | LabelledContent | None]
66
+
67
+ def __post_init__(self):
68
+ self.paragraphs = [p for p in self.paragraphs if p is not None]
69
+
70
+ def text(self, recipient) -> str:
71
+ return render_template(
72
+ "mail/message.txt",
73
+ message=self,
74
+ recipient=recipient,
75
+ )
76
+
77
+ def html(self, recipient) -> str:
78
+ return render_template(
79
+ "mail/message.html",
80
+ message=self,
81
+ recipient=recipient,
82
+ )
47
83
 
48
- tpl_path = f"mail/{template_base}"
84
+ def send(self, recipients):
85
+ send_mail(recipients, self)
49
86
 
87
+
88
+ def init_app(app):
89
+ mail.init_app(app)
90
+
91
+
92
+ def send_mail(recipients: object | list, message: MailMessage):
50
93
  debug = current_app.config.get("DEBUG", False)
51
94
  send_mail = current_app.config.get("SEND_MAIL", not debug)
52
- connection = mail.connect if send_mail else dummyconnection
53
- extras = get_mail_campaign_dict()
54
-
55
- with connection() as conn:
56
- for recipient in recipients:
57
- lang = i18n._default_lang(recipient)
58
- with i18n.language(lang):
59
- log.debug('Sending mail "%s" to recipient "%s"', subject, recipient)
60
- msg = Message(subject, sender=sender, recipients=[recipient.email])
61
- msg.body = render_template(
62
- f"{tpl_path}.txt",
63
- subject=subject,
64
- sender=sender,
65
- recipient=recipient,
66
- extras=extras,
67
- **kwargs,
68
- )
69
- msg.html = render_template(
70
- f"{tpl_path}.html",
71
- subject=subject,
72
- sender=sender,
73
- recipient=recipient,
74
- extras=extras,
75
- **kwargs,
76
- )
77
- try:
78
- conn.send(msg)
79
- except SMTPException as e:
80
- log.error(f"Error sending mail {e}")
95
+
96
+ if not isinstance(recipients, list):
97
+ recipients = [recipients]
98
+
99
+ for recipient in recipients:
100
+ lang = i18n._default_lang(recipient)
101
+ to = recipient if isinstance(recipient, str) else recipient.email
102
+ with i18n.language(lang):
103
+ msg = Message(
104
+ subject=str(message.subject),
105
+ body=message.text(recipient),
106
+ html=message.html(recipient),
107
+ recipients=[to],
108
+ )
109
+
110
+ if send_mail:
111
+ with mail.connect() as conn:
112
+ conn.send(msg)
113
+ else:
114
+ log.debug(f"Sending mail {message.subject} to {to}")
115
+ log.debug(msg.body)
116
+ log.debug(msg.html)
117
+ mail_sent.send(msg)
81
118
 
82
119
 
83
120
  def get_mail_campaign_dict() -> dict:
@@ -6,11 +6,7 @@ import logging
6
6
 
7
7
  from mongoengine.connection import get_db
8
8
 
9
- from udata.core.dataservices.constants import (
10
- DATASERVICE_ACCESS_TYPE_OPEN,
11
- DATASERVICE_ACCESS_TYPE_OPEN_WITH_ACCOUNT,
12
- DATASERVICE_ACCESS_TYPE_RESTRICTED,
13
- )
9
+ from udata.core.access_type.constants import AccessType
14
10
  from udata.core.dataservices.models import Dataservice
15
11
 
16
12
  log = logging.getLogger(__name__)
@@ -47,7 +43,7 @@ def migrate(db):
47
43
 
48
44
  for dataservice in get_db().dataservice.find({"is_restricted": True, "has_token": False}):
49
45
  log.info(
50
- f"\tDataservice #{dataservice['_id']} {dataservice['title']} is restricted but without token. (will be set to access_type={DATASERVICE_ACCESS_TYPE_RESTRICTED})"
46
+ f"\tDataservice #{dataservice['_id']} {dataservice['title']} is restricted but without token. (will be set to access_type={AccessType.RESTRICTED})"
51
47
  )
52
48
 
53
49
  log.info("Processing dataservices…")
@@ -57,21 +53,19 @@ def migrate(db):
57
53
  "is_restricted": True,
58
54
  # `has_token` could be True or False, we don't care
59
55
  },
60
- update={"$set": {"access_type": DATASERVICE_ACCESS_TYPE_RESTRICTED}},
61
- )
62
- log.info(
63
- f"\t{count.modified_count} restricted dataservices to DATASERVICE_ACCESS_TYPE_RESTRICTED"
56
+ update={"$set": {"access_type": AccessType.RESTRICTED}},
64
57
  )
58
+ log.info(f"\t{count.modified_count} restricted dataservices to AccessType.RESTRICTED")
65
59
 
66
60
  count = get_db().dataservice.update_many(
67
61
  filter={
68
62
  "is_restricted": False,
69
63
  "has_token": True,
70
64
  },
71
- update={"$set": {"access_type": DATASERVICE_ACCESS_TYPE_OPEN_WITH_ACCOUNT}},
65
+ update={"$set": {"access_type": AccessType.OPEN_WITH_ACCOUNT}},
72
66
  )
73
67
  log.info(
74
- f"\t{count.modified_count} dataservices not restricted but with token to DATASERVICE_ACCESS_TYPE_OPEN_WITH_ACCOUNT"
68
+ f"\t{count.modified_count} dataservices not restricted but with token to AccessType.OPEN_WITH_ACCOUNT"
75
69
  )
76
70
 
77
71
  count = get_db().dataservice.update_many(
@@ -79,9 +73,9 @@ def migrate(db):
79
73
  "is_restricted": False,
80
74
  "has_token": False,
81
75
  },
82
- update={"$set": {"access_type": DATASERVICE_ACCESS_TYPE_OPEN}},
76
+ update={"$set": {"access_type": AccessType.OPEN}},
83
77
  )
84
- log.info(f"\t{count.modified_count} open dataservices to DATASERVICE_ACCESS_TYPE_OPEN")
78
+ log.info(f"\t{count.modified_count} open dataservices to AccessType.OPEN")
85
79
 
86
80
  dataservices: list[Dataservice] = get_db().dataservice.find()
87
81
  for dataservice in dataservices:
@@ -0,0 +1,28 @@
1
+ """
2
+ This migration empties harvest.modified_at field in the case of CKAN datasets.
3
+ Indeed, the value that was stored in this field was the *metadata* modification data
4
+ and not the *data* one, contrary to other backends.
5
+ """
6
+
7
+ import logging
8
+
9
+ import click
10
+
11
+ from udata.core.dataset.models import Dataset
12
+
13
+ log = logging.getLogger(__name__)
14
+
15
+
16
+ def migrate(db):
17
+ datasets = Dataset.objects(harvest__backend="CKAN", harvest__modified_at__exists=True)
18
+ count = datasets.count()
19
+
20
+ with click.progressbar(datasets, length=count) as datasets:
21
+ for dataset in datasets:
22
+ dataset.harvest.modified_at = None
23
+ try:
24
+ dataset.save()
25
+ except Exception as err:
26
+ log.error(f"Cannot save dataset {dataset.id} {err}")
27
+ log.info(f"Updated {count} datasets")
28
+ log.info("Done")
@@ -0,0 +1,27 @@
1
+ """
2
+ Fix HarvestSource with removed `periodic_task`
3
+ """
4
+
5
+ import logging
6
+
7
+ import mongoengine
8
+
9
+ from udata.harvest.models import HarvestSource
10
+
11
+ log = logging.getLogger(__name__)
12
+
13
+
14
+ def migrate(db):
15
+ log.info("Processing HarvestSource periodic_task.")
16
+
17
+ sources = HarvestSource.objects
18
+ count = 0
19
+ for source in sources:
20
+ try:
21
+ source.schedule # query periodic_task
22
+ except mongoengine.errors.DoesNotExist:
23
+ count += 1
24
+ source.periodic_task = None
25
+ source.save()
26
+
27
+ log.info(f"Modified {count} sources")
@@ -32,12 +32,12 @@ class TagListField(ListField):
32
32
  super(TagListField, self).validate(values)
33
33
 
34
34
  for tag in values:
35
- if not tags.MIN_TAG_LENGTH <= len(tag) <= tags.MAX_TAG_LENGTH:
35
+ if not tags.TAG_MIN_LENGTH <= len(tag) <= tags.TAG_MAX_LENGTH:
36
36
  self.error(
37
37
  _(
38
38
  'Tag "%(tag)s" must be between %(min)d and %(max)d characters long.',
39
- min=tags.MIN_TAG_LENGTH,
40
- max=tags.MAX_TAG_LENGTH,
39
+ min=tags.TAG_MIN_LENGTH,
40
+ max=tags.TAG_MAX_LENGTH,
41
41
  tag=tag,
42
42
  )
43
43
  )
udata/rdf.py CHANGED
@@ -136,15 +136,21 @@ INSPIRE_GEMET_SCHEME_URIS = [
136
136
 
137
137
  AGENT_ROLE_TO_RDF_PREDICATE = {
138
138
  "contact": DCAT.contactPoint,
139
- "publisher": DCT.publisher,
140
139
  "creator": DCT.creator,
140
+ "publisher": DCT.publisher,
141
+ "rightsHolder": DCT.rightsHolder,
142
+ "custodian": GEODCAT.custodian,
143
+ "distributor": GEODCAT.distributor,
144
+ "originator": GEODCAT.originator,
145
+ "principalInvestigator": GEODCAT.principalInvestigator,
146
+ "processor": GEODCAT.processor,
147
+ "resourceProvider": GEODCAT.resourceProvider,
148
+ "user": GEODCAT.user,
141
149
  }
142
150
 
143
151
  # Map rdf contact point entity to role
144
152
  CONTACT_POINT_ENTITY_TO_ROLE = {
145
- DCAT.contactPoint: "contact",
146
- DCT.publisher: "publisher",
147
- DCT.creator: "creator",
153
+ predicate: role for role, predicate in AGENT_ROLE_TO_RDF_PREDICATE.items()
148
154
  }
149
155
 
150
156
 
@@ -353,6 +359,8 @@ def themes_from_rdf(rdf):
353
359
 
354
360
 
355
361
  def contact_points_from_rdf(rdf, prop, role, dataset):
362
+ if not dataset.organization and not dataset.owner:
363
+ return
356
364
  for contact_point in rdf.objects(prop):
357
365
  # Read contact point information
358
366
  if isinstance(contact_point, Literal):
@@ -365,7 +373,7 @@ def contact_points_from_rdf(rdf, prop, role, dataset):
365
373
  email = (
366
374
  rdf_value(contact_point, VCARD.hasEmail)
367
375
  or rdf_value(contact_point, VCARD.email)
368
- or rdf_value(contact_point, DCAT.email)
376
+ or None
369
377
  )
370
378
  email = email.replace("mailto:", "").strip() if email else None
371
379
  contact_form = rdf_value(contact_point, VCARD.hasUrl)
@@ -384,8 +392,6 @@ def contact_points_from_rdf(rdf, prop, role, dataset):
384
392
  # continue
385
393
 
386
394
  # Create of get contact point object
387
- if not dataset.organization and not dataset.owner:
388
- continue
389
395
  org_or_owner = {}
390
396
  if dataset.organization:
391
397
  org_or_owner = {"organization": dataset.organization}
@@ -420,14 +426,25 @@ def contact_points_to_rdf(contacts, graph=None):
420
426
  id = BNode()
421
427
 
422
428
  node = graph.resource(id)
423
- node.set(RDF.type, VCARD.Kind)
424
- if contact.name:
425
- node.set(VCARD.fn, Literal(contact.name))
426
- if contact.email:
427
- node.set(VCARD.hasEmail, URIRef(f"mailto:{contact.email}"))
428
- if contact.contact_form:
429
- node.set(VCARD.hasUrl, URIRef(contact.contact_form))
430
- yield node, AGENT_ROLE_TO_RDF_PREDICATE.get(contact.role, DCAT.contactPoint)
429
+ role = AGENT_ROLE_TO_RDF_PREDICATE.get(contact.role, DCAT.contactPoint)
430
+ # GeoDCAT-AP spec: Only contactPoint is a VCARD.Kind (like in DCAT). Other roles are FOAF.Agent.
431
+ if role == DCAT.contactPoint:
432
+ node.set(RDF.type, VCARD.Kind)
433
+ if contact.name:
434
+ node.set(VCARD.fn, Literal(contact.name))
435
+ if contact.email:
436
+ node.set(VCARD.hasEmail, URIRef(f"mailto:{contact.email}"))
437
+ if contact.contact_form:
438
+ node.set(VCARD.hasUrl, URIRef(contact.contact_form))
439
+ else:
440
+ node.set(RDF.type, FOAF.Agent)
441
+ node.set(FOAF.name, Literal(contact.name))
442
+ if contact.email:
443
+ node.set(FOAF.mbox, URIRef(f"mailto:{contact.email}"))
444
+ if contact.contact_form:
445
+ node.set(FOAF.page, URIRef(contact.contact_form))
446
+
447
+ yield node, role
431
448
 
432
449
 
433
450
  def primary_topic_identifier_from_rdf(graph: Graph, resource: RdfResource):