udata 12.0.2.dev10__py3-none-any.whl → 13.0.1.dev21__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of udata might be problematic. Click here for more details.

Files changed (272) hide show
  1. udata/api/__init__.py +1 -0
  2. udata/api_fields.py +10 -4
  3. udata/app.py +11 -10
  4. udata/auth/__init__.py +9 -10
  5. udata/auth/mails.py +137 -45
  6. udata/auth/views.py +5 -12
  7. udata/commands/__init__.py +2 -4
  8. udata/commands/info.py +1 -3
  9. udata/commands/tests/test_fixtures.py +6 -3
  10. udata/core/access_type/api.py +18 -0
  11. udata/core/access_type/constants.py +98 -0
  12. udata/core/access_type/models.py +44 -0
  13. udata/core/activity/models.py +1 -1
  14. udata/core/badges/models.py +1 -1
  15. udata/core/badges/tasks.py +35 -1
  16. udata/core/badges/tests/test_commands.py +2 -4
  17. udata/core/badges/tests/test_model.py +2 -2
  18. udata/core/badges/tests/test_tasks.py +55 -0
  19. udata/core/constants.py +1 -0
  20. udata/core/contact_point/models.py +8 -0
  21. udata/core/dataservices/api.py +10 -12
  22. udata/core/dataservices/apiv2.py +3 -1
  23. udata/core/dataservices/constants.py +0 -29
  24. udata/core/dataservices/models.py +44 -44
  25. udata/core/dataservices/rdf.py +2 -1
  26. udata/core/dataservices/search.py +5 -9
  27. udata/core/dataservices/tasks.py +33 -0
  28. udata/core/dataset/api.py +15 -24
  29. udata/core/dataset/api_fields.py +11 -0
  30. udata/core/dataset/apiv2.py +11 -0
  31. udata/core/dataset/constants.py +0 -1
  32. udata/core/dataset/forms.py +29 -0
  33. udata/core/dataset/models.py +24 -42
  34. udata/core/dataset/rdf.py +2 -1
  35. udata/core/dataset/search.py +2 -2
  36. udata/core/dataset/tasks.py +86 -8
  37. udata/core/discussions/mails.py +63 -0
  38. udata/core/discussions/tasks.py +4 -18
  39. udata/core/metrics/__init__.py +0 -6
  40. udata/core/organization/api.py +20 -14
  41. udata/core/organization/mails.py +144 -0
  42. udata/core/organization/models.py +2 -1
  43. udata/core/organization/rdf.py +3 -3
  44. udata/core/organization/search.py +1 -1
  45. udata/core/organization/tasks.py +21 -49
  46. udata/core/pages/tests/test_api.py +0 -2
  47. udata/core/reuse/api.py +29 -3
  48. udata/core/reuse/mails.py +21 -0
  49. udata/core/reuse/models.py +10 -1
  50. udata/core/reuse/search.py +1 -1
  51. udata/core/reuse/tasks.py +2 -3
  52. udata/core/site/api.py +27 -19
  53. udata/core/site/models.py +2 -6
  54. udata/core/site/rdf.py +2 -2
  55. udata/core/spatial/tests/test_api.py +17 -20
  56. udata/core/spatial/tests/test_models.py +3 -3
  57. udata/core/user/mails.py +54 -0
  58. udata/core/user/models.py +2 -3
  59. udata/core/user/tasks.py +8 -23
  60. udata/core/user/tests/test_user_model.py +2 -6
  61. udata/entrypoints.py +0 -6
  62. udata/features/identicon/tests/test_backends.py +3 -13
  63. udata/forms/fields.py +3 -3
  64. udata/forms/widgets.py +2 -2
  65. udata/frontend/__init__.py +3 -32
  66. udata/harvest/actions.py +4 -9
  67. udata/harvest/api.py +5 -14
  68. udata/harvest/backends/__init__.py +20 -11
  69. udata/harvest/backends/base.py +2 -2
  70. udata/harvest/backends/ckan/harvesters.py +2 -1
  71. udata/harvest/backends/dcat.py +3 -0
  72. udata/harvest/backends/maaf.py +1 -0
  73. udata/harvest/commands.py +6 -4
  74. udata/harvest/forms.py +9 -6
  75. udata/harvest/tasks.py +3 -5
  76. udata/harvest/tests/ckan/test_ckan_backend.py +300 -337
  77. udata/harvest/tests/ckan/test_ckan_backend_errors.py +94 -99
  78. udata/harvest/tests/ckan/test_ckan_backend_filters.py +128 -122
  79. udata/harvest/tests/ckan/test_dkan_backend.py +39 -51
  80. udata/harvest/tests/dcat/bnodes.xml +17 -1
  81. udata/harvest/tests/dcat/datara--5a26b0f6-0ccf-46ad-ac58-734054b91977.rdf.xml +255 -0
  82. udata/harvest/tests/dcat/datara--f40c3860-7236-4b30-a141-23b8ae33f7b2.rdf.xml +289 -0
  83. udata/harvest/tests/factories.py +1 -1
  84. udata/harvest/tests/test_actions.py +11 -9
  85. udata/harvest/tests/test_api.py +4 -5
  86. udata/harvest/tests/test_base_backend.py +5 -4
  87. udata/harvest/tests/test_dcat_backend.py +72 -16
  88. udata/harvest/tests/test_models.py +2 -4
  89. udata/harvest/tests/test_notifications.py +2 -4
  90. udata/harvest/tests/test_tasks.py +2 -3
  91. udata/mail.py +90 -53
  92. udata/migrations/2025-01-05-dataservices-fields-changes.py +8 -14
  93. udata/migrations/2025-10-21-remove-ckan-harvest-modified-at.py +28 -0
  94. udata/migrations/2025-10-29-harvesters-sources-integrity.py +27 -0
  95. udata/models/__init__.py +0 -2
  96. udata/mongo/extras_fields.py +4 -3
  97. udata/mongo/taglist_field.py +3 -3
  98. udata/rdf.py +65 -20
  99. udata/sentry.py +3 -4
  100. udata/settings.py +15 -13
  101. udata/tags.py +5 -5
  102. udata/tasks.py +3 -3
  103. udata/templates/mail/message.html +65 -0
  104. udata/templates/mail/message.txt +16 -0
  105. udata/tests/__init__.py +40 -58
  106. udata/tests/api/__init__.py +87 -2
  107. udata/tests/api/test_activities_api.py +17 -23
  108. udata/tests/api/test_auth_api.py +2 -4
  109. udata/tests/api/test_contact_points.py +48 -54
  110. udata/tests/api/test_dataservices_api.py +65 -97
  111. udata/tests/api/test_datasets_api.py +171 -56
  112. udata/tests/api/test_me_api.py +4 -6
  113. udata/tests/api/test_organizations_api.py +19 -38
  114. udata/tests/api/test_reports_api.py +0 -4
  115. udata/tests/api/test_reuses_api.py +99 -23
  116. udata/tests/api/test_security_api.py +124 -0
  117. udata/tests/api/test_swagger.py +2 -3
  118. udata/tests/api/test_tags_api.py +6 -7
  119. udata/tests/api/test_transfer_api.py +0 -2
  120. udata/tests/api/test_user_api.py +8 -10
  121. udata/tests/apiv2/test_datasets.py +0 -4
  122. udata/tests/apiv2/test_me_api.py +0 -2
  123. udata/tests/apiv2/test_organizations.py +0 -2
  124. udata/tests/apiv2/test_swagger.py +2 -3
  125. udata/tests/apiv2/test_topics.py +0 -2
  126. udata/tests/cli/test_cli_base.py +14 -12
  127. udata/tests/cli/test_db_cli.py +51 -54
  128. udata/tests/contact_point/test_contact_point_models.py +2 -2
  129. udata/tests/dataservice/test_csv_adapter.py +2 -5
  130. udata/tests/dataservice/test_dataservice_rdf.py +64 -4
  131. udata/tests/dataservice/test_dataservice_tasks.py +36 -38
  132. udata/tests/dataset/test_csv_adapter.py +2 -5
  133. udata/tests/dataset/test_dataset_actions.py +2 -4
  134. udata/tests/dataset/test_dataset_commands.py +2 -4
  135. udata/tests/dataset/test_dataset_events.py +3 -3
  136. udata/tests/dataset/test_dataset_model.py +6 -7
  137. udata/tests/dataset/test_dataset_rdf.py +205 -16
  138. udata/tests/dataset/test_dataset_recommendations.py +2 -2
  139. udata/tests/dataset/test_dataset_tasks.py +66 -68
  140. udata/tests/dataset/test_resource_preview.py +39 -48
  141. udata/tests/dataset/test_transport_tasks.py +2 -2
  142. udata/tests/features/territories/__init__.py +0 -6
  143. udata/tests/features/territories/test_territories_api.py +25 -24
  144. udata/tests/forms/test_current_user_field.py +2 -2
  145. udata/tests/forms/test_dict_field.py +2 -4
  146. udata/tests/forms/test_extras_fields.py +2 -3
  147. udata/tests/forms/test_image_field.py +2 -2
  148. udata/tests/forms/test_model_field.py +2 -4
  149. udata/tests/forms/test_publish_as_field.py +2 -4
  150. udata/tests/forms/test_user_forms.py +26 -29
  151. udata/tests/frontend/test_auth.py +2 -3
  152. udata/tests/frontend/test_csv.py +5 -6
  153. udata/tests/frontend/test_error_handlers.py +2 -3
  154. udata/tests/frontend/test_hooks.py +5 -7
  155. udata/tests/frontend/test_markdown.py +3 -4
  156. udata/tests/helpers.py +2 -7
  157. udata/tests/metrics/test_metrics.py +52 -48
  158. udata/tests/metrics/test_tasks.py +154 -150
  159. udata/tests/organization/test_csv_adapter.py +2 -5
  160. udata/tests/organization/test_notifications.py +2 -4
  161. udata/tests/organization/test_organization_model.py +3 -4
  162. udata/tests/organization/test_organization_rdf.py +6 -12
  163. udata/tests/plugin.py +6 -110
  164. udata/tests/reuse/test_reuse_model.py +3 -4
  165. udata/tests/site/test_site_api.py +0 -2
  166. udata/tests/site/test_site_csv_exports.py +0 -2
  167. udata/tests/site/test_site_metrics.py +2 -4
  168. udata/tests/site/test_site_model.py +2 -2
  169. udata/tests/site/test_site_rdf.py +85 -29
  170. udata/tests/test_activity.py +3 -3
  171. udata/tests/test_api_fields.py +6 -9
  172. udata/tests/test_cors.py +0 -2
  173. udata/tests/test_dcat_commands.py +2 -3
  174. udata/tests/test_discussions.py +2 -7
  175. udata/tests/test_mail.py +150 -114
  176. udata/tests/test_migrations.py +413 -419
  177. udata/tests/test_model.py +10 -11
  178. udata/tests/test_notifications.py +2 -3
  179. udata/tests/test_owned.py +3 -3
  180. udata/tests/test_rdf.py +19 -15
  181. udata/tests/test_routing.py +5 -5
  182. udata/tests/test_storages.py +6 -5
  183. udata/tests/test_tags.py +2 -4
  184. udata/tests/test_topics.py +2 -4
  185. udata/tests/test_transfer.py +4 -5
  186. udata/tests/topic/test_topic_tasks.py +25 -27
  187. udata/tests/user/test_user_rdf.py +2 -8
  188. udata/tests/user/test_user_tasks.py +3 -5
  189. udata/tests/workers/test_jobs_commands.py +2 -2
  190. udata/tests/workers/test_tasks_routing.py +27 -27
  191. udata/translations/ar/LC_MESSAGES/udata.mo +0 -0
  192. udata/translations/ar/LC_MESSAGES/udata.po +369 -435
  193. udata/translations/de/LC_MESSAGES/udata.mo +0 -0
  194. udata/translations/de/LC_MESSAGES/udata.po +371 -437
  195. udata/translations/es/LC_MESSAGES/udata.mo +0 -0
  196. udata/translations/es/LC_MESSAGES/udata.po +369 -435
  197. udata/translations/fr/LC_MESSAGES/udata.mo +0 -0
  198. udata/translations/fr/LC_MESSAGES/udata.po +381 -447
  199. udata/translations/it/LC_MESSAGES/udata.mo +0 -0
  200. udata/translations/it/LC_MESSAGES/udata.po +371 -437
  201. udata/translations/pt/LC_MESSAGES/udata.mo +0 -0
  202. udata/translations/pt/LC_MESSAGES/udata.po +371 -437
  203. udata/translations/sr/LC_MESSAGES/udata.mo +0 -0
  204. udata/translations/sr/LC_MESSAGES/udata.po +372 -438
  205. udata/translations/udata.pot +379 -440
  206. udata/utils.py +66 -4
  207. {udata-12.0.2.dev10.dist-info → udata-13.0.1.dev21.dist-info}/METADATA +1 -4
  208. {udata-12.0.2.dev10.dist-info → udata-13.0.1.dev21.dist-info}/RECORD +212 -256
  209. udata/linkchecker/__init__.py +0 -0
  210. udata/linkchecker/backends.py +0 -31
  211. udata/linkchecker/checker.py +0 -75
  212. udata/linkchecker/commands.py +0 -21
  213. udata/linkchecker/models.py +0 -9
  214. udata/linkchecker/tasks.py +0 -55
  215. udata/templates/mail/account_deleted.html +0 -5
  216. udata/templates/mail/account_deleted.txt +0 -6
  217. udata/templates/mail/account_inactivity.html +0 -40
  218. udata/templates/mail/account_inactivity.txt +0 -31
  219. udata/templates/mail/badge_added_association.html +0 -33
  220. udata/templates/mail/badge_added_association.txt +0 -11
  221. udata/templates/mail/badge_added_certified.html +0 -33
  222. udata/templates/mail/badge_added_certified.txt +0 -11
  223. udata/templates/mail/badge_added_company.html +0 -33
  224. udata/templates/mail/badge_added_company.txt +0 -11
  225. udata/templates/mail/badge_added_local_authority.html +0 -33
  226. udata/templates/mail/badge_added_local_authority.txt +0 -11
  227. udata/templates/mail/badge_added_public_service.html +0 -33
  228. udata/templates/mail/badge_added_public_service.txt +0 -11
  229. udata/templates/mail/discussion_closed.html +0 -47
  230. udata/templates/mail/discussion_closed.txt +0 -16
  231. udata/templates/mail/inactive_account_deleted.html +0 -5
  232. udata/templates/mail/inactive_account_deleted.txt +0 -6
  233. udata/templates/mail/membership_refused.html +0 -20
  234. udata/templates/mail/membership_refused.txt +0 -11
  235. udata/templates/mail/membership_request.html +0 -46
  236. udata/templates/mail/membership_request.txt +0 -12
  237. udata/templates/mail/new_discussion.html +0 -44
  238. udata/templates/mail/new_discussion.txt +0 -15
  239. udata/templates/mail/new_discussion_comment.html +0 -45
  240. udata/templates/mail/new_discussion_comment.txt +0 -16
  241. udata/templates/mail/new_member.html +0 -27
  242. udata/templates/mail/new_member.txt +0 -11
  243. udata/templates/mail/new_reuse.html +0 -37
  244. udata/templates/mail/new_reuse.txt +0 -9
  245. udata/templates/mail/test.html +0 -6
  246. udata/templates/mail/test.txt +0 -6
  247. udata/templates/mail/user_mail_card.html +0 -26
  248. udata/templates/security/email/base.html +0 -105
  249. udata/templates/security/email/base.txt +0 -6
  250. udata/templates/security/email/button.html +0 -3
  251. udata/templates/security/email/change_notice.html +0 -22
  252. udata/templates/security/email/change_notice.txt +0 -8
  253. udata/templates/security/email/confirmation_instructions.html +0 -20
  254. udata/templates/security/email/confirmation_instructions.txt +0 -7
  255. udata/templates/security/email/login_instructions.html +0 -19
  256. udata/templates/security/email/login_instructions.txt +0 -7
  257. udata/templates/security/email/reset_instructions.html +0 -24
  258. udata/templates/security/email/reset_instructions.txt +0 -9
  259. udata/templates/security/email/reset_notice.html +0 -11
  260. udata/templates/security/email/reset_notice.txt +0 -4
  261. udata/templates/security/email/welcome.html +0 -24
  262. udata/templates/security/email/welcome.txt +0 -9
  263. udata/templates/security/email/welcome_existing.html +0 -32
  264. udata/templates/security/email/welcome_existing.txt +0 -14
  265. udata/terms.md +0 -6
  266. udata/tests/frontend/__init__.py +0 -23
  267. udata/tests/metrics/conftest.py +0 -15
  268. udata/tests/test_linkchecker.py +0 -277
  269. {udata-12.0.2.dev10.dist-info → udata-13.0.1.dev21.dist-info}/WHEEL +0 -0
  270. {udata-12.0.2.dev10.dist-info → udata-13.0.1.dev21.dist-info}/entry_points.txt +0 -0
  271. {udata-12.0.2.dev10.dist-info → udata-13.0.1.dev21.dist-info}/licenses/LICENSE +0 -0
  272. {udata-12.0.2.dev10.dist-info → udata-13.0.1.dev21.dist-info}/top_level.txt +0 -0
udata/harvest/tests/ckan/test_ckan_backend_errors.py CHANGED
@@ -2,11 +2,7 @@ import pytest
2
2
 
3
3
  from udata.harvest import actions
4
4
  from udata.harvest.tests.factories import HarvestSourceFactory
5
-
6
- pytestmark = [
7
- pytest.mark.usefixtures("clean_db"),
8
- pytest.mark.options(PLUGINS=["ckan"]),
9
- ]
5
+ from udata.tests.api import PytestOnlyDBTestCase
10
6
 
11
7
  CKAN_URL = "https://harvest.me/"
12
8
  API_URL = "{}api/3/action/package_list".format(CKAN_URL)
@@ -17,124 +13,123 @@ API_URL = "{}api/3/action/package_list".format(CKAN_URL)
17
13
  STATUS_CODE = (400, 500)
18
14
 
19
15
 
20
- @pytest.mark.parametrize("code", STATUS_CODE)
21
- def test_html_error(rmock, code):
22
- # Happens with wrong source URL (html is returned instead of json)
23
- html = "<html><body>Error</body></html>"
24
- source = HarvestSourceFactory(backend="ckan", url=CKAN_URL)
25
-
26
- rmock.get(API_URL, text=html, status_code=code, headers={"Content-Type": "text/html"})
27
-
28
- actions.run(source)
29
-
30
- source.reload()
31
-
32
- job = source.get_last_job()
33
- assert len(job.items) == 0
34
- assert len(job.errors) == 1
35
- error = job.errors[0]
36
- # HTML is detected and does not clutter the message
37
- assert html not in error.message
38
-
39
-
40
- @pytest.mark.parametrize("code", STATUS_CODE)
41
- def test_plain_text_error(rmock, code):
42
- source = HarvestSourceFactory(backend="ckan", url=CKAN_URL)
43
-
44
- rmock.get(
45
- API_URL, text='"Some error"', status_code=code, headers={"Content-Type": "text/plain"}
46
- )
16
+ @pytest.mark.options(HARVESTER_BACKENDS=["ckan"])
17
+ class CkanBackendErrorsTest(PytestOnlyDBTestCase):
18
+ @pytest.mark.parametrize("code", STATUS_CODE)
19
+ def test_html_error(self, rmock, code):
20
+ # Happens with wrong source URL (html is returned instead of json)
21
+ html = "<html><body>Error</body></html>"
22
+ source = HarvestSourceFactory(backend="ckan", url=CKAN_URL)
47
23
 
48
- actions.run(source)
24
+ rmock.get(API_URL, text=html, status_code=code, headers={"Content-Type": "text/html"})
49
25
 
50
- source.reload()
26
+ actions.run(source)
51
27
 
52
- job = source.get_last_job()
53
- assert len(job.items) == 0
54
- assert len(job.errors) == 1
55
- error = job.errors[0]
56
- # Raw quoted string is properly unquoted
57
- http_message = "Server Error" if code == 500 else "Client Error"
58
- assert (
59
- error.message
60
- == f"{code} {http_message}: None for url: https://harvest.me/api/3/action/package_list"
61
- )
28
+ source.reload()
62
29
 
30
+ job = source.get_last_job()
31
+ assert len(job.items) == 0
32
+ assert len(job.errors) == 1
33
+ error = job.errors[0]
34
+ # HTML is detected and does not clutter the message
35
+ assert html not in error.message
63
36
 
64
- def test_200_plain_text_error(rmock):
65
- source = HarvestSourceFactory(backend="ckan", url=CKAN_URL)
37
+ @pytest.mark.parametrize("code", STATUS_CODE)
38
+ def test_plain_text_error(self, rmock, code):
39
+ source = HarvestSourceFactory(backend="ckan", url=CKAN_URL)
66
40
 
67
- rmock.get(API_URL, text='"Some error"', status_code=200, headers={"Content-Type": "text/plain"})
41
+ rmock.get(
42
+ API_URL, text='"Some error"', status_code=code, headers={"Content-Type": "text/plain"}
43
+ )
68
44
 
69
- actions.run(source)
45
+ actions.run(source)
70
46
 
71
- source.reload()
47
+ source.reload()
72
48
 
73
- job = source.get_last_job()
74
- assert len(job.items) == 0
75
- assert len(job.errors) == 1
76
- error = job.errors[0]
77
- # Raw quoted string is properly unquoted
78
- assert error.message == "Some error"
49
+ job = source.get_last_job()
50
+ assert len(job.items) == 0
51
+ assert len(job.errors) == 1
52
+ error = job.errors[0]
53
+ # Raw quoted string is properly unquoted
54
+ http_message = "Server Error" if code == 500 else "Client Error"
55
+ assert (
56
+ error.message
57
+ == f"{code} {http_message}: None for url: https://harvest.me/api/3/action/package_list"
58
+ )
79
59
 
60
+ def test_200_plain_text_error(self, rmock):
61
+ source = HarvestSourceFactory(backend="ckan", url=CKAN_URL)
80
62
 
81
- def test_standard_api_json_error(rmock):
82
- json = {"success": False, "error": "an error"}
83
- source = HarvestSourceFactory(backend="ckan", url=CKAN_URL)
63
+ rmock.get(
64
+ API_URL, text='"Some error"', status_code=200, headers={"Content-Type": "text/plain"}
65
+ )
84
66
 
85
- rmock.get(API_URL, json=json, status_code=200, headers={"Content-Type": "application/json"})
67
+ actions.run(source)
86
68
 
87
- actions.run(source)
69
+ source.reload()
88
70
 
89
- source.reload()
71
+ job = source.get_last_job()
72
+ assert len(job.items) == 0
73
+ assert len(job.errors) == 1
74
+ error = job.errors[0]
75
+ # Raw quoted string is properly unquoted
76
+ assert error.message == "Some error"
90
77
 
91
- job = source.get_last_job()
92
- assert len(job.items) == 0
93
- assert len(job.errors) == 1
94
- error = job.errors[0]
95
- assert error.message == "an error"
78
+ def test_standard_api_json_error(self, rmock):
79
+ json = {"success": False, "error": "an error"}
80
+ source = HarvestSourceFactory(backend="ckan", url=CKAN_URL)
96
81
 
82
+ rmock.get(API_URL, json=json, status_code=200, headers={"Content-Type": "application/json"})
97
83
 
98
- def test_standard_api_json_error_with_details(rmock):
99
- json = {
100
- "success": False,
101
- "error": {
102
- "message": "an error",
103
- },
104
- }
105
- source = HarvestSourceFactory(backend="ckan", url=CKAN_URL)
84
+ actions.run(source)
106
85
 
107
- rmock.get(API_URL, json=json, status_code=200, headers={"Content-Type": "application/json"})
86
+ source.reload()
108
87
 
109
- actions.run(source)
88
+ job = source.get_last_job()
89
+ assert len(job.items) == 0
90
+ assert len(job.errors) == 1
91
+ error = job.errors[0]
92
+ assert error.message == "an error"
110
93
 
111
- source.reload()
94
+ def test_standard_api_json_error_with_details(self, rmock):
95
+ json = {
96
+ "success": False,
97
+ "error": {
98
+ "message": "an error",
99
+ },
100
+ }
101
+ source = HarvestSourceFactory(backend="ckan", url=CKAN_URL)
112
102
 
113
- job = source.get_last_job()
114
- assert len(job.items) == 0
115
- assert len(job.errors) == 1
116
- error = job.errors[0]
117
- assert error.message == "an error"
103
+ rmock.get(API_URL, json=json, status_code=200, headers={"Content-Type": "application/json"})
118
104
 
105
+ actions.run(source)
119
106
 
120
- def test_standard_api_json_error_with_details_and_type(rmock):
121
- json = {
122
- "success": False,
123
- "error": {
124
- "message": "Access denied",
125
- "__type": "Authorization Error",
126
- },
127
- }
128
- source = HarvestSourceFactory(backend="ckan", url=CKAN_URL)
107
+ source.reload()
129
108
 
130
- rmock.get(API_URL, json=json, status_code=200, headers={"Content-Type": "application/json"})
109
+ job = source.get_last_job()
110
+ assert len(job.items) == 0
111
+ assert len(job.errors) == 1
112
+ error = job.errors[0]
113
+ assert error.message == "an error"
131
114
 
132
- actions.run(source)
115
+ def test_standard_api_json_error_with_details_and_type(self, rmock):
116
+ json = {
117
+ "success": False,
118
+ "error": {
119
+ "message": "Access denied",
120
+ "__type": "Authorization Error",
121
+ },
122
+ }
123
+ source = HarvestSourceFactory(backend="ckan", url=CKAN_URL)
133
124
 
134
- source.reload()
125
+ rmock.get(API_URL, json=json, status_code=200, headers={"Content-Type": "application/json"})
135
126
 
136
- job = source.get_last_job()
137
- assert len(job.items) == 0
138
- assert len(job.errors) == 1
139
- error = job.errors[0]
140
- assert error.message == "Authorization Error: Access denied"
127
+ actions.run(source)
128
+
129
+ source.reload()
130
+
131
+ job = source.get_last_job()
132
+ assert len(job.items) == 0
133
+ assert len(job.errors) == 1
134
+ error = job.errors[0]
135
+ assert error.message == "Authorization Error: Access denied"
udata/harvest/tests/ckan/test_ckan_backend_filters.py CHANGED
@@ -4,127 +4,133 @@ import pytest
4
4
 
5
5
  from udata.harvest import actions
6
6
  from udata.harvest.tests.factories import HarvestSourceFactory
7
+ from udata.tests.api import PytestOnlyDBTestCase
7
8
  from udata.utils import faker
8
9
 
9
- pytestmark = [
10
- pytest.mark.usefixtures("clean_db"),
11
- pytest.mark.options(PLUGINS=["ckan"]),
12
- ]
13
-
14
-
15
- def test_include_org_filter(ckan, rmock):
16
- source = HarvestSourceFactory(
17
- backend="ckan",
18
- url=ckan.BASE_URL,
19
- config={"filters": [{"key": "organization", "value": "organization_name"}]},
20
- )
21
-
22
- rmock.get(
23
- ckan.PACKAGE_SEARCH_URL,
24
- json={"success": True, "result": {"results": []}},
25
- status_code=200,
26
- headers={"Content-Type": "application/json"},
27
- )
28
-
29
- actions.run(source)
30
- source.reload()
31
-
32
- assert rmock.call_count == 1
33
- params = {"q": "organization:organization_name", "rows": 1000}
34
- assert rmock.last_request.url == f"{ckan.PACKAGE_SEARCH_URL}?{urllib.parse.urlencode(params)}"
35
-
36
-
37
- def test_exclude_org_filter(ckan, rmock):
38
- source = HarvestSourceFactory(
39
- backend="ckan",
40
- url=ckan.BASE_URL,
41
- config={
42
- "filters": [{"key": "organization", "value": "organization_name", "type": "exclude"}]
43
- },
44
- )
45
-
46
- rmock.get(
47
- ckan.PACKAGE_SEARCH_URL,
48
- json={"success": True, "result": {"results": []}},
49
- status_code=200,
50
- headers={"Content-Type": "application/json"},
51
- )
52
-
53
- actions.run(source)
54
- source.reload()
55
-
56
- assert rmock.call_count == 1
57
-
58
- params = {"q": "-organization:organization_name", "rows": 1000}
59
- assert rmock.last_request.url == f"{ckan.PACKAGE_SEARCH_URL}?{urllib.parse.urlencode(params)}"
60
-
61
-
62
- def test_tag_filter(ckan, rmock):
63
- tag = faker.word()
64
- source = HarvestSourceFactory(
65
- backend="ckan", url=ckan.BASE_URL, config={"filters": [{"key": "tags", "value": tag}]}
66
- )
67
-
68
- rmock.get(
69
- ckan.PACKAGE_SEARCH_URL,
70
- json={"success": True, "result": {"results": []}},
71
- status_code=200,
72
- headers={"Content-Type": "application/json"},
73
- )
74
-
75
- actions.run(source)
76
- source.reload()
77
-
78
- assert rmock.call_count == 1
79
- params = {"q": f"tags:{tag}", "rows": 1000}
80
- assert rmock.last_request.url == f"{ckan.PACKAGE_SEARCH_URL}?{urllib.parse.urlencode(params)}"
81
-
82
-
83
- def test_exclude_tag_filter(ckan, rmock):
84
- tag = faker.word()
85
- source = HarvestSourceFactory(
86
- backend="ckan",
87
- url=ckan.BASE_URL,
88
- config={"filters": [{"key": "tags", "value": tag, "type": "exclude"}]},
89
- )
90
-
91
- rmock.get(
92
- ckan.PACKAGE_SEARCH_URL,
93
- json={"success": True, "result": {"results": []}},
94
- status_code=200,
95
- headers={"Content-Type": "application/json"},
96
- )
97
-
98
- actions.run(source)
99
- source.reload()
100
-
101
- assert rmock.call_count == 1
102
- params = {"q": f"-tags:{tag}", "rows": 1000}
103
- assert rmock.last_request.url == f"{ckan.PACKAGE_SEARCH_URL}?{urllib.parse.urlencode(params)}"
104
-
105
-
106
- def test_can_have_multiple_filters(ckan, rmock):
107
- source = HarvestSourceFactory(
108
- backend="ckan",
109
- url=ckan.BASE_URL,
110
- config={
111
- "filters": [
112
- {"key": "organization", "value": "organization_name"},
113
- {"key": "tags", "value": "tag-2", "type": "exclude"},
114
- ]
115
- },
116
- )
117
-
118
- rmock.get(
119
- ckan.PACKAGE_SEARCH_URL,
120
- json={"success": True, "result": {"results": []}},
121
- status_code=200,
122
- headers={"Content-Type": "application/json"},
123
- )
124
-
125
- actions.run(source)
126
- source.reload()
127
-
128
- assert rmock.call_count == 1
129
- params = {"q": "organization:organization_name AND -tags:tag-2", "rows": 1000}
130
- assert rmock.last_request.url == f"{ckan.PACKAGE_SEARCH_URL}?{urllib.parse.urlencode(params)}"
10
+
11
+ @pytest.mark.options(HARVESTER_BACKENDS=["ckan"])
12
+ class CkanBackendFilterTest(PytestOnlyDBTestCase):
13
+ def test_include_org_filter(self, ckan, rmock):
14
+ source = HarvestSourceFactory(
15
+ backend="ckan",
16
+ url=ckan.BASE_URL,
17
+ config={"filters": [{"key": "organization", "value": "organization_name"}]},
18
+ )
19
+
20
+ rmock.get(
21
+ ckan.PACKAGE_SEARCH_URL,
22
+ json={"success": True, "result": {"results": []}},
23
+ status_code=200,
24
+ headers={"Content-Type": "application/json"},
25
+ )
26
+
27
+ actions.run(source)
28
+ source.reload()
29
+
30
+ assert rmock.call_count == 1
31
+ params = {"q": "organization:organization_name", "rows": 1000}
32
+ assert (
33
+ rmock.last_request.url == f"{ckan.PACKAGE_SEARCH_URL}?{urllib.parse.urlencode(params)}"
34
+ )
35
+
36
+ def test_exclude_org_filter(self, ckan, rmock):
37
+ source = HarvestSourceFactory(
38
+ backend="ckan",
39
+ url=ckan.BASE_URL,
40
+ config={
41
+ "filters": [
42
+ {"key": "organization", "value": "organization_name", "type": "exclude"}
43
+ ]
44
+ },
45
+ )
46
+
47
+ rmock.get(
48
+ ckan.PACKAGE_SEARCH_URL,
49
+ json={"success": True, "result": {"results": []}},
50
+ status_code=200,
51
+ headers={"Content-Type": "application/json"},
52
+ )
53
+
54
+ actions.run(source)
55
+ source.reload()
56
+
57
+ assert rmock.call_count == 1
58
+
59
+ params = {"q": "-organization:organization_name", "rows": 1000}
60
+ assert (
61
+ rmock.last_request.url == f"{ckan.PACKAGE_SEARCH_URL}?{urllib.parse.urlencode(params)}"
62
+ )
63
+
64
+ def test_tag_filter(self, ckan, rmock):
65
+ tag = faker.word()
66
+ source = HarvestSourceFactory(
67
+ backend="ckan", url=ckan.BASE_URL, config={"filters": [{"key": "tags", "value": tag}]}
68
+ )
69
+
70
+ rmock.get(
71
+ ckan.PACKAGE_SEARCH_URL,
72
+ json={"success": True, "result": {"results": []}},
73
+ status_code=200,
74
+ headers={"Content-Type": "application/json"},
75
+ )
76
+
77
+ actions.run(source)
78
+ source.reload()
79
+
80
+ assert rmock.call_count == 1
81
+ params = {"q": f"tags:{tag}", "rows": 1000}
82
+ assert (
83
+ rmock.last_request.url == f"{ckan.PACKAGE_SEARCH_URL}?{urllib.parse.urlencode(params)}"
84
+ )
85
+
86
+ def test_exclude_tag_filter(self, ckan, rmock):
87
+ tag = faker.word()
88
+ source = HarvestSourceFactory(
89
+ backend="ckan",
90
+ url=ckan.BASE_URL,
91
+ config={"filters": [{"key": "tags", "value": tag, "type": "exclude"}]},
92
+ )
93
+
94
+ rmock.get(
95
+ ckan.PACKAGE_SEARCH_URL,
96
+ json={"success": True, "result": {"results": []}},
97
+ status_code=200,
98
+ headers={"Content-Type": "application/json"},
99
+ )
100
+
101
+ actions.run(source)
102
+ source.reload()
103
+
104
+ assert rmock.call_count == 1
105
+ params = {"q": f"-tags:{tag}", "rows": 1000}
106
+ assert (
107
+ rmock.last_request.url == f"{ckan.PACKAGE_SEARCH_URL}?{urllib.parse.urlencode(params)}"
108
+ )
109
+
110
+ def test_can_have_multiple_filters(self, ckan, rmock):
111
+ source = HarvestSourceFactory(
112
+ backend="ckan",
113
+ url=ckan.BASE_URL,
114
+ config={
115
+ "filters": [
116
+ {"key": "organization", "value": "organization_name"},
117
+ {"key": "tags", "value": "tag-2", "type": "exclude"},
118
+ ]
119
+ },
120
+ )
121
+
122
+ rmock.get(
123
+ ckan.PACKAGE_SEARCH_URL,
124
+ json={"success": True, "result": {"results": []}},
125
+ status_code=200,
126
+ headers={"Content-Type": "application/json"},
127
+ )
128
+
129
+ actions.run(source)
130
+ source.reload()
131
+
132
+ assert rmock.call_count == 1
133
+ params = {"q": "organization:organization_name AND -tags:tag-2", "rows": 1000}
134
+ assert (
135
+ rmock.last_request.url == f"{ckan.PACKAGE_SEARCH_URL}?{urllib.parse.urlencode(params)}"
136
+ )
udata/harvest/tests/ckan/test_dkan_backend.py CHANGED
@@ -4,13 +4,11 @@ from datetime import datetime
4
4
 
5
5
  import pytest
6
6
 
7
- from udata.app import create_app
8
7
  from udata.core.organization.factories import OrganizationFactory
9
8
  from udata.harvest import actions
10
9
  from udata.harvest.tests.factories import HarvestSourceFactory
11
10
  from udata.models import Dataset
12
- from udata.settings import Defaults, Testing
13
- from udata.tests.plugin import drop_db
11
+ from udata.tests.api import PytestOnlyDBTestCase
14
12
 
15
13
 
16
14
  def data_path(filename):
@@ -18,51 +16,41 @@ def data_path(filename):
18
16
  return os.path.join(os.path.dirname(__file__), "data", filename)
19
17
 
20
18
 
21
- class DkanSettings(Testing):
22
- PLUGINS = ["dkan"]
23
-
24
-
25
- @pytest.fixture(scope="module")
26
- def app(request):
27
- """Create an udata app once for the module."""
28
- app = create_app(Defaults, override=DkanSettings)
29
- with app.app_context():
30
- drop_db(app)
31
- yield app
32
- with app.app_context():
33
- drop_db(app)
34
-
35
-
36
- def test_dkan_french_w_license(app, rmock):
37
- """CKAN Harvester should accept the minimum dataset payload"""
38
- DKAN_URL = "https://harvest.me/"
39
- API_URL = "{}api/3/action/".format(DKAN_URL)
40
- PACKAGE_LIST_URL = "{}package_list".format(API_URL)
41
- PACKAGE_SHOW_URL = "{}package_show".format(API_URL)
42
-
43
- with open(data_path("dkan-french-w-license.json")) as ifile:
44
- data = json.loads(ifile.read())
45
-
46
- org = OrganizationFactory()
47
- source = HarvestSourceFactory(backend="dkan", url=DKAN_URL, organization=org)
48
- rmock.get(
49
- PACKAGE_LIST_URL,
50
- json={"success": True, "result": ["fake-name"]},
51
- status_code=200,
52
- headers={"Content-Type": "application/json"},
53
- )
54
- rmock.get(
55
- PACKAGE_SHOW_URL, json=data, status_code=200, headers={"Content-Type": "application/json"}
56
- )
57
- actions.run(source)
58
- source.reload()
59
- assert source.get_last_job().status == "done"
60
-
61
- datasets = Dataset.objects.filter(organization=org)
62
- assert len(datasets) > 0
63
-
64
- dataset = datasets.get(**{"harvest__remote_id": "04be6288-696d-4331-850d-a144871a7e3a"})
65
- assert dataset.harvest.created_at == datetime(2019, 12, 10, 0, 0)
66
- assert dataset.harvest.modified_at == datetime(2019, 9, 30, 0, 0)
67
- assert len(dataset.resources) == 2
68
- assert "xlsx" in [r.format for r in dataset.resources]
19
+ @pytest.mark.options(HARVESTER_BACKENDS=["dkan"])
20
+ class DkanBackendTest(PytestOnlyDBTestCase):
21
+ def test_dkan_french_w_license(self, rmock):
22
+ """CKAN Harvester should accept the minimum dataset payload"""
23
+ DKAN_URL = "https://harvest.me/"
24
+ API_URL = "{}api/3/action/".format(DKAN_URL)
25
+ PACKAGE_LIST_URL = "{}package_list".format(API_URL)
26
+ PACKAGE_SHOW_URL = "{}package_show".format(API_URL)
27
+
28
+ with open(data_path("dkan-french-w-license.json")) as ifile:
29
+ data = json.loads(ifile.read())
30
+
31
+ org = OrganizationFactory()
32
+ source = HarvestSourceFactory(backend="dkan", url=DKAN_URL, organization=org)
33
+ rmock.get(
34
+ PACKAGE_LIST_URL,
35
+ json={"success": True, "result": ["fake-name"]},
36
+ status_code=200,
37
+ headers={"Content-Type": "application/json"},
38
+ )
39
+ rmock.get(
40
+ PACKAGE_SHOW_URL,
41
+ json=data,
42
+ status_code=200,
43
+ headers={"Content-Type": "application/json"},
44
+ )
45
+ actions.run(source)
46
+ source.reload()
47
+ assert source.get_last_job().status == "done"
48
+
49
+ datasets = Dataset.objects.filter(organization=org)
50
+ assert len(datasets) > 0
51
+
52
+ dataset = datasets.get(**{"harvest__remote_id": "04be6288-696d-4331-850d-a144871a7e3a"})
53
+ assert dataset.harvest.created_at == datetime(2019, 12, 10, 0, 0)
54
+ assert dataset.harvest.modified_at is None
55
+ assert len(dataset.resources) == 2
56
+ assert "xlsx" in [r.format for r in dataset.resources]
udata/harvest/tests/dcat/bnodes.xml CHANGED
@@ -11,6 +11,7 @@
11
11
  xmlns:dcterms="http://purl.org/dc/terms/"
12
12
  xmlns:vcard="http://www.w3.org/2006/vcard/ns#"
13
13
  xmlns:schema="http://schema.org/"
14
+ xmlns:skos="http://www.w3.org/2004/02/skos/core#"
14
15
  >
15
16
  <dcat:Catalog rdf:about="http://data.test.org/">
16
17
  <dcat:dataset>
@@ -45,7 +46,6 @@
45
46
  <dcterms:title>Sample DCAT Catalog</dcterms:title>
46
47
  <dcat:dataset>
47
48
  <dcat:Dataset>
48
- <dcat:theme>Theme 2</dcat:theme>
49
49
  <dcat:contactPoint rdf:resource="http://data.test.org/contacts/1"/>
50
50
  <dcat:landingPage>http://data.test.org/datasets/1</dcat:landingPage>
51
51
  <dcat:keyword>Tag 3</dcat:keyword>
@@ -75,6 +75,17 @@
75
75
  <dcat:distribution rdf:resource="http://data.test.org/datasets/1/resources/1"/>
76
76
  <dcterms:issued rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2016-12-14T18:59:02.737480</dcterms:issued>
77
77
  <dcterms:identifier>1</dcterms:identifier>
78
+ <dcat:theme>
79
+ <skos:Concept>
80
+ <skos:prefLabel xml:lang="fr">Répartition des espèces</skos:prefLabel>
81
+ <skos:inScheme>
82
+ <skos:ConceptScheme>
83
+ <dcterms:title xml:lang="fr">GEMET - INSPIRE themes, version 1.0</dcterms:title>
84
+ <dcterms:issued rdf:datatype="http://www.w3.org/2001/XMLSchema#date">2018-07-27</dcterms:issued>
85
+ </skos:ConceptScheme>
86
+ </skos:inScheme>
87
+ </skos:Concept>
88
+ </dcat:theme>
78
89
  </dcat:Dataset>
79
90
  </dcat:dataset>
80
91
  <dcat:dataset>
@@ -110,6 +121,7 @@
110
121
  </dct:spatial>
111
122
  <dcterms:identifier>2</dcterms:identifier>
112
123
  <dct:conformsTo rdf:nodeID="Ne0189e93917c4f67a412fc44883322e7"/>
124
+ <dcat:theme rdf:resource="http://bnode.namespace.voc/theme/hy"/>
113
125
  </dcat:Dataset>
114
126
  </dcat:dataset>
115
127
  <dcat:service>
@@ -185,4 +197,8 @@
185
197
  <dct:type rdf:resource="http://inspire.ec.europa.eu/glossary/SpatialReferenceSystem"/>
186
198
  <dct:title xml:lang="fr">RGF93 / Lambert-93 (EPSG:2154)</dct:title>
187
199
  </rdf:Description>
200
+ <skos:Concept rdf:about="http://bnode.namespace.voc/theme/hy">
201
+ <skos:inScheme rdf:resource="http://inspire.ec.europa.eu/theme"/>
202
+ <skos:prefLabel>Hydrographie</skos:prefLabel>
203
+ </skos:Concept>
188
204
  </rdf:RDF>