udata 13.0.1.dev12__py3-none-any.whl → 14.4.1.dev7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of udata might be problematic. Click here for more details.
- udata/api/__init__.py +2 -8
- udata/api_fields.py +35 -4
- udata/app.py +30 -50
- udata/auth/__init__.py +29 -6
- udata/auth/forms.py +8 -6
- udata/auth/views.py +6 -3
- udata/commands/__init__.py +2 -14
- udata/commands/db.py +13 -25
- udata/commands/info.py +0 -16
- udata/commands/serve.py +3 -11
- udata/commands/tests/test_fixtures.py +9 -9
- udata/core/access_type/api.py +1 -1
- udata/core/access_type/constants.py +12 -8
- udata/core/activity/api.py +5 -6
- udata/core/avatars/api.py +43 -0
- udata/core/avatars/test_avatar_api.py +30 -0
- udata/core/badges/tests/test_commands.py +6 -6
- udata/core/csv.py +5 -0
- udata/core/dataservices/models.py +15 -3
- udata/core/dataservices/tasks.py +7 -0
- udata/core/dataset/api.py +2 -0
- udata/core/dataset/models.py +2 -2
- udata/core/dataset/permissions.py +31 -0
- udata/core/dataset/tasks.py +50 -10
- udata/core/discussions/models.py +1 -0
- udata/core/metrics/__init__.py +0 -6
- udata/core/organization/api.py +8 -5
- udata/core/organization/mails.py +1 -1
- udata/core/organization/models.py +9 -1
- udata/core/organization/notifications.py +84 -0
- udata/core/organization/permissions.py +1 -1
- udata/core/organization/tasks.py +3 -0
- udata/core/pages/tests/test_api.py +32 -0
- udata/core/post/api.py +24 -69
- udata/core/post/models.py +84 -16
- udata/core/post/tests/test_api.py +24 -1
- udata/core/reports/api.py +18 -0
- udata/core/reports/models.py +42 -2
- udata/core/reuse/models.py +1 -1
- udata/core/reuse/tasks.py +7 -0
- udata/core/site/models.py +2 -6
- udata/core/spatial/commands.py +2 -4
- udata/core/spatial/forms.py +2 -2
- udata/core/spatial/models.py +0 -10
- udata/core/spatial/tests/test_api.py +1 -36
- udata/core/user/models.py +15 -2
- udata/cors.py +2 -5
- udata/db/migrations.py +279 -0
- udata/features/notifications/api.py +7 -18
- udata/features/notifications/models.py +56 -0
- udata/features/notifications/tasks.py +25 -0
- udata/flask_mongoengine/engine.py +0 -4
- udata/frontend/__init__.py +3 -122
- udata/frontend/markdown.py +2 -1
- udata/harvest/actions.py +24 -9
- udata/harvest/api.py +30 -22
- udata/harvest/backends/__init__.py +21 -9
- udata/harvest/backends/base.py +29 -3
- udata/harvest/backends/ckan/harvesters.py +13 -2
- udata/harvest/backends/dcat.py +3 -0
- udata/harvest/backends/maaf.py +1 -0
- udata/harvest/commands.py +39 -4
- udata/harvest/filters.py +17 -6
- udata/harvest/forms.py +9 -6
- udata/harvest/models.py +16 -0
- udata/harvest/permissions.py +27 -0
- udata/harvest/tasks.py +3 -5
- udata/harvest/tests/ckan/test_ckan_backend.py +35 -2
- udata/harvest/tests/ckan/test_ckan_backend_errors.py +1 -1
- udata/harvest/tests/ckan/test_ckan_backend_filters.py +1 -1
- udata/harvest/tests/ckan/test_dkan_backend.py +1 -1
- udata/harvest/tests/dcat/udata.xml +6 -6
- udata/harvest/tests/factories.py +1 -1
- udata/harvest/tests/test_actions.py +63 -8
- udata/harvest/tests/test_api.py +278 -123
- udata/harvest/tests/test_base_backend.py +88 -1
- udata/harvest/tests/test_dcat_backend.py +60 -13
- udata/harvest/tests/test_filters.py +6 -0
- udata/i18n.py +11 -273
- udata/mail.py +5 -1
- udata/migrations/2025-10-31-create-membership-request-notifications.py +55 -0
- udata/migrations/2025-11-13-delete-user-email-index.py +25 -0
- udata/migrations/2025-12-04-add-uuid-to-discussion-messages.py +28 -0
- udata/models/__init__.py +0 -8
- udata/mongo/slug_fields.py +1 -1
- udata/rdf.py +45 -6
- udata/routing.py +2 -10
- udata/sentry.py +4 -10
- udata/settings.py +23 -17
- udata/tasks.py +4 -3
- udata/templates/mail/message.html +5 -31
- udata/tests/__init__.py +28 -12
- udata/tests/api/__init__.py +108 -21
- udata/tests/api/test_activities_api.py +36 -0
- udata/tests/api/test_auth_api.py +121 -95
- udata/tests/api/test_base_api.py +7 -4
- udata/tests/api/test_dataservices_api.py +29 -1
- udata/tests/api/test_datasets_api.py +45 -21
- udata/tests/api/test_organizations_api.py +192 -197
- udata/tests/api/test_reports_api.py +157 -0
- udata/tests/api/test_reuses_api.py +147 -147
- udata/tests/api/test_security_api.py +12 -12
- udata/tests/api/test_swagger.py +4 -4
- udata/tests/api/test_tags_api.py +8 -8
- udata/tests/api/test_user_api.py +13 -1
- udata/tests/apiv2/test_swagger.py +4 -4
- udata/tests/apiv2/test_topics.py +1 -1
- udata/tests/cli/test_cli_base.py +8 -9
- udata/tests/dataset/test_dataset_commands.py +4 -4
- udata/tests/dataset/test_dataset_model.py +66 -26
- udata/tests/dataset/test_dataset_rdf.py +99 -5
- udata/tests/dataset/test_resource_preview.py +0 -1
- udata/tests/frontend/test_auth.py +24 -1
- udata/tests/frontend/test_csv.py +0 -3
- udata/tests/helpers.py +37 -27
- udata/tests/organization/test_notifications.py +67 -2
- udata/tests/plugin.py +6 -261
- udata/tests/site/test_site_csv_exports.py +22 -10
- udata/tests/test_activity.py +9 -9
- udata/tests/test_cors.py +1 -1
- udata/tests/test_dcat_commands.py +2 -2
- udata/tests/test_discussions.py +5 -5
- udata/tests/test_migrations.py +181 -481
- udata/tests/test_notifications.py +15 -57
- udata/tests/test_notifications_task.py +43 -0
- udata/tests/test_owned.py +81 -1
- udata/tests/test_storages.py +25 -19
- udata/tests/test_topics.py +77 -61
- udata/tests/test_uris.py +33 -0
- udata/tests/workers/test_jobs_commands.py +23 -23
- udata/translations/ar/LC_MESSAGES/udata.mo +0 -0
- udata/translations/ar/LC_MESSAGES/udata.po +187 -108
- udata/translations/de/LC_MESSAGES/udata.mo +0 -0
- udata/translations/de/LC_MESSAGES/udata.po +187 -108
- udata/translations/es/LC_MESSAGES/udata.mo +0 -0
- udata/translations/es/LC_MESSAGES/udata.po +187 -108
- udata/translations/fr/LC_MESSAGES/udata.mo +0 -0
- udata/translations/fr/LC_MESSAGES/udata.po +188 -109
- udata/translations/it/LC_MESSAGES/udata.mo +0 -0
- udata/translations/it/LC_MESSAGES/udata.po +187 -108
- udata/translations/pt/LC_MESSAGES/udata.mo +0 -0
- udata/translations/pt/LC_MESSAGES/udata.po +187 -108
- udata/translations/sr/LC_MESSAGES/udata.mo +0 -0
- udata/translations/sr/LC_MESSAGES/udata.po +187 -108
- udata/translations/udata.pot +215 -106
- udata/uris.py +0 -2
- udata/utils.py +5 -0
- udata-14.4.1.dev7.dist-info/METADATA +109 -0
- {udata-13.0.1.dev12.dist-info → udata-14.4.1.dev7.dist-info}/RECORD +153 -166
- {udata-13.0.1.dev12.dist-info → udata-14.4.1.dev7.dist-info}/entry_points.txt +3 -5
- udata/core/followers/views.py +0 -15
- udata/core/post/forms.py +0 -30
- udata/entrypoints.py +0 -93
- udata/features/identicon/__init__.py +0 -0
- udata/features/identicon/api.py +0 -13
- udata/features/identicon/backends.py +0 -131
- udata/features/identicon/tests/__init__.py +0 -0
- udata/features/identicon/tests/test_backends.py +0 -18
- udata/features/territories/__init__.py +0 -49
- udata/features/territories/api.py +0 -25
- udata/features/territories/models.py +0 -51
- udata/flask_mongoengine/json.py +0 -38
- udata/migrations/__init__.py +0 -367
- udata/templates/mail/base.html +0 -105
- udata/templates/mail/base.txt +0 -6
- udata/templates/mail/button.html +0 -3
- udata/templates/mail/layouts/1-column.html +0 -19
- udata/templates/mail/layouts/2-columns.html +0 -20
- udata/templates/mail/layouts/center-panel.html +0 -16
- udata/tests/cli/test_db_cli.py +0 -68
- udata/tests/features/territories/__init__.py +0 -20
- udata/tests/features/territories/test_territories_api.py +0 -185
- udata/tests/frontend/test_hooks.py +0 -149
- udata-13.0.1.dev12.dist-info/METADATA +0 -133
- {udata-13.0.1.dev12.dist-info → udata-14.4.1.dev7.dist-info}/WHEEL +0 -0
- {udata-13.0.1.dev12.dist-info → udata-14.4.1.dev7.dist-info}/licenses/LICENSE +0 -0
- {udata-13.0.1.dev12.dist-info → udata-14.4.1.dev7.dist-info}/top_level.txt +0 -0
|
@@ -24,7 +24,7 @@ from udata.tests.api import PytestOnlyAPITestCase
|
|
|
24
24
|
|
|
25
25
|
class FixturesTest(PytestOnlyAPITestCase):
|
|
26
26
|
@pytest.mark.options(FIXTURE_DATASET_SLUGS=["some-test-dataset-slug"])
|
|
27
|
-
def test_generate_fixtures_file_then_import(self,
|
|
27
|
+
def test_generate_fixtures_file_then_import(self, mocker):
|
|
28
28
|
"""Test generating fixtures from the current env, then importing them back."""
|
|
29
29
|
assert models.Dataset.objects.count() == 0 # Start with a clean slate.
|
|
30
30
|
user = UserFactory()
|
|
@@ -55,11 +55,11 @@ class FixturesTest(PytestOnlyAPITestCase):
|
|
|
55
55
|
DataserviceFactory(datasets=[dataset], organization=org, contact_points=[contact_point])
|
|
56
56
|
|
|
57
57
|
with NamedTemporaryFile(mode="w+", delete=True) as fixtures_fd:
|
|
58
|
-
# Get the fixtures from the local instance.
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
Response
|
|
62
|
-
result = cli("generate-fixtures-file", "", fixtures_fd.name)
|
|
58
|
+
# Get the fixtures from the local instance by redirecting requests.get to the test client
|
|
59
|
+
mocker.patch.object(requests, "get", side_effect=lambda url: self.get(url))
|
|
60
|
+
mocker.patch.object(Response, "json", Response.get_json)
|
|
61
|
+
mocker.patch.object(Response, "ok", True, create=True)
|
|
62
|
+
result = self.cli("generate-fixtures-file", "", fixtures_fd.name)
|
|
63
63
|
fixtures_fd.flush()
|
|
64
64
|
assert "Fixtures saved to file " in result.output
|
|
65
65
|
|
|
@@ -81,7 +81,7 @@ class FixturesTest(PytestOnlyAPITestCase):
|
|
|
81
81
|
assert models.ContactPoint.objects.count() == 0
|
|
82
82
|
|
|
83
83
|
# Then load them in the database to make sure they're correct.
|
|
84
|
-
result = cli("import-fixtures", fixtures_fd.name)
|
|
84
|
+
result = self.cli("import-fixtures", fixtures_fd.name)
|
|
85
85
|
assert models.Organization.objects(slug=org.slug).count() > 0
|
|
86
86
|
result_org = models.Organization.objects.get(slug=org.slug)
|
|
87
87
|
assert result_org.members[0].user.id == user.id
|
|
@@ -106,11 +106,11 @@ class FixturesTest(PytestOnlyAPITestCase):
|
|
|
106
106
|
assert result_dataservice.organization == org
|
|
107
107
|
assert result_dataservice.contact_points == [contact_point]
|
|
108
108
|
|
|
109
|
-
def test_import_fixtures_from_default_file(self
|
|
109
|
+
def test_import_fixtures_from_default_file(self):
|
|
110
110
|
"""Test importing fixtures from udata.commands.fixture.DEFAULT_FIXTURE_FILE."""
|
|
111
111
|
# Deactivate spam detection when testing import fixtures
|
|
112
112
|
SpamMixin.detect_spam_enabled = False
|
|
113
|
-
cli("import-fixtures")
|
|
113
|
+
self.cli("import-fixtures")
|
|
114
114
|
SpamMixin.detect_spam_enabled = True
|
|
115
115
|
assert models.Organization.objects.count() > 0
|
|
116
116
|
assert models.Dataset.objects.count() > 0
|
udata/core/access_type/api.py
CHANGED
|
@@ -13,6 +13,6 @@ class ReasonCategoriesAPI(API):
|
|
|
13
13
|
def get(self):
|
|
14
14
|
"""List all limitation reason categories"""
|
|
15
15
|
return [
|
|
16
|
-
{"value": category.value, "label": category.label}
|
|
16
|
+
{"value": category.value, "label": category.label, "definition": category.definition}
|
|
17
17
|
for category in InspireLimitationCategory
|
|
18
18
|
]
|
|
@@ -63,31 +63,35 @@ class InspireLimitationCategory(StrEnum):
|
|
|
63
63
|
match self:
|
|
64
64
|
case InspireLimitationCategory.PUBLIC_AUTHORITIES:
|
|
65
65
|
return _(
|
|
66
|
-
"
|
|
66
|
+
"Public access to datasets and services would adversely affect the confidentiality of the proceedings of public authorities, where such confidentiality is provided for by law."
|
|
67
67
|
)
|
|
68
68
|
case InspireLimitationCategory.INTERNATIONAL_RELATIONS:
|
|
69
|
-
return _(
|
|
69
|
+
return _(
|
|
70
|
+
"Public access to datasets and services would adversely affect international relations, public security or national defence."
|
|
71
|
+
)
|
|
70
72
|
case InspireLimitationCategory.COURSE_OF_JUSTICE:
|
|
71
73
|
return _(
|
|
72
|
-
"
|
|
74
|
+
"Public access to datasets and services would adversely affect the course of justice, the ability of any person to receive a fair trial or the ability of a public authority to conduct an enquiry of a criminal or disciplinary nature."
|
|
73
75
|
)
|
|
74
76
|
case InspireLimitationCategory.COMMERCIAL_CONFIDENTIALITY:
|
|
75
77
|
return _(
|
|
76
|
-
"
|
|
78
|
+
"Public access to datasets and services would adversely affect the confidentiality of commercial or industrial information, where such confidentiality is provided for by national or Community law to protect a legitimate economic interest, including the public interest in maintaining statistical confidentiality and tax secrecy."
|
|
77
79
|
)
|
|
78
80
|
case InspireLimitationCategory.INTELLECTUAL_PROPERTY:
|
|
79
|
-
return _(
|
|
81
|
+
return _(
|
|
82
|
+
"Public access to datasets and services would adversely affect intellectual property rights."
|
|
83
|
+
)
|
|
80
84
|
case InspireLimitationCategory.PERSONAL_DATA:
|
|
81
85
|
return _(
|
|
82
|
-
"
|
|
86
|
+
"Public access to datasets and services would adversely affect the confidentiality of personal data and/or files relating to a natural person where that person has not consented to the disclosure of the information to the public, where such confidentiality is provided for by national or Community law."
|
|
83
87
|
)
|
|
84
88
|
case InspireLimitationCategory.VOLUNTARY_SUPPLIER:
|
|
85
89
|
return _(
|
|
86
|
-
"
|
|
90
|
+
"Public access to datasets and services would adversely affect the interests or protection of any person who supplied the information requested on a voluntary basis without being under, or capable of being put under, a legal obligation to do so, unless that person has consented to the release of the information concerned."
|
|
87
91
|
)
|
|
88
92
|
case InspireLimitationCategory.ENVIRONMENTAL_PROTECTION:
|
|
89
93
|
return _(
|
|
90
|
-
"
|
|
94
|
+
"Public access to datasets and services would adversely affect the protection of the environment to which such information relates, such as the location of rare species."
|
|
91
95
|
)
|
|
92
96
|
case _:
|
|
93
97
|
assert_never(self)
|
udata/core/activity/api.py
CHANGED
|
@@ -4,8 +4,9 @@ from bson import ObjectId
|
|
|
4
4
|
from mongoengine.errors import DoesNotExist
|
|
5
5
|
|
|
6
6
|
from udata.api import API, api, fields
|
|
7
|
-
from udata.
|
|
7
|
+
from udata.core.dataset.permissions import OwnableReadPermission
|
|
8
8
|
from udata.core.organization.api_fields import org_ref_fields
|
|
9
|
+
from udata.core.owned import Owned
|
|
9
10
|
from udata.core.user.api_fields import user_ref_fields
|
|
10
11
|
from udata.models import Activity, db
|
|
11
12
|
|
|
@@ -101,7 +102,7 @@ class SiteActivityAPI(API):
|
|
|
101
102
|
# Always return a result even not complete
|
|
102
103
|
# But log the error (ie. visible in sentry, silent for user)
|
|
103
104
|
# Can happen when someone manually delete an object in DB (ie. without proper purge)
|
|
104
|
-
# - Filter out
|
|
105
|
+
# - Filter out items not visible to the current user
|
|
105
106
|
safe_items = []
|
|
106
107
|
for item in qs.queryset.items:
|
|
107
108
|
try:
|
|
@@ -109,10 +110,8 @@ class SiteActivityAPI(API):
|
|
|
109
110
|
except DoesNotExist as e:
|
|
110
111
|
log.error(e, exc_info=True)
|
|
111
112
|
else:
|
|
112
|
-
if
|
|
113
|
-
|
|
114
|
-
):
|
|
115
|
-
if item.related_to.private:
|
|
113
|
+
if isinstance(item.related_to, Owned):
|
|
114
|
+
if not OwnableReadPermission(item.related_to).can():
|
|
116
115
|
continue
|
|
117
116
|
safe_items.append(item)
|
|
118
117
|
qs.queryset.items = safe_items
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import hashlib
|
|
2
|
+
import io
|
|
3
|
+
|
|
4
|
+
import pydenticon
|
|
5
|
+
from flask import current_app, send_file
|
|
6
|
+
|
|
7
|
+
from udata.api import API, api
|
|
8
|
+
from udata.app import cache
|
|
9
|
+
|
|
10
|
+
ns = api.namespace("avatars", "Avatars")
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@cache.memoize()
|
|
14
|
+
def generate_pydenticon(identifier, size):
|
|
15
|
+
"""
|
|
16
|
+
Use pydenticon to generate an identicon image.
|
|
17
|
+
All parameters are extracted from configuration.
|
|
18
|
+
"""
|
|
19
|
+
blocks_size = current_app.config["AVATAR_INTERNAL_SIZE"]
|
|
20
|
+
foreground = current_app.config["AVATAR_INTERNAL_FOREGROUND"]
|
|
21
|
+
background = current_app.config["AVATAR_INTERNAL_BACKGROUND"]
|
|
22
|
+
generator = pydenticon.Generator(
|
|
23
|
+
blocks_size, blocks_size, digest=hashlib.sha1, foreground=foreground, background=background
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
# Pydenticon adds padding to the size and as a consequence
|
|
27
|
+
# we need to compute the size without the padding
|
|
28
|
+
padding = int(round(current_app.config["AVATAR_INTERNAL_PADDING"] * size / 100.0))
|
|
29
|
+
size = size - 2 * padding
|
|
30
|
+
padding = (padding,) * 4
|
|
31
|
+
return generator.generate(identifier, size, size, padding=padding, output_format="png")
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@ns.route("/<identifier>/<int:size>/", endpoint="avatar")
|
|
35
|
+
class IdenticonAPI(API):
|
|
36
|
+
@api.doc("avatars")
|
|
37
|
+
def get(self, identifier, size):
|
|
38
|
+
"""Get a deterministic avatar given an identifier at a given size"""
|
|
39
|
+
identicon = generate_pydenticon(identifier, size)
|
|
40
|
+
response = send_file(io.BytesIO(identicon), mimetype="image/png")
|
|
41
|
+
etag = hashlib.sha1(identicon).hexdigest()
|
|
42
|
+
response.set_etag(etag)
|
|
43
|
+
return response
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
from flask import url_for
|
|
2
|
+
|
|
3
|
+
from udata.tests.api import PytestOnlyAPITestCase
|
|
4
|
+
from udata.tests.helpers import assert200
|
|
5
|
+
from udata.utils import faker
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def assert_stream_equal(response1, response2):
|
|
9
|
+
__tracebackhide__ = True
|
|
10
|
+
stream1 = list(response1.iter_encoded())
|
|
11
|
+
stream2 = list(response2.iter_encoded())
|
|
12
|
+
assert stream1 == stream2
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class InternalBackendTest(PytestOnlyAPITestCase):
|
|
16
|
+
def test_base_rendering(self):
|
|
17
|
+
response = self.get(url_for("api.avatar", identifier=faker.word(), size=32))
|
|
18
|
+
|
|
19
|
+
assert200(response)
|
|
20
|
+
assert response.mimetype == "image/png"
|
|
21
|
+
assert response.is_streamed
|
|
22
|
+
etag, weak = response.get_etag()
|
|
23
|
+
assert etag is not None
|
|
24
|
+
|
|
25
|
+
def test_render_twice_the_same(self):
|
|
26
|
+
identifier = faker.word()
|
|
27
|
+
stream_a = self.get(url_for("api.avatar", identifier=identifier, size=32))
|
|
28
|
+
stream_b = self.get(url_for("api.avatar", identifier=identifier, size=32))
|
|
29
|
+
|
|
30
|
+
assert_stream_equal(stream_a, stream_b)
|
|
@@ -9,32 +9,32 @@ class BadgeCommandTest(PytestOnlyDBTestCase):
|
|
|
9
9
|
def toggle(self, path_or_id, kind):
|
|
10
10
|
return self.cli("badges", "toggle", path_or_id, kind)
|
|
11
11
|
|
|
12
|
-
def test_toggle_badge_on(self
|
|
12
|
+
def test_toggle_badge_on(self):
|
|
13
13
|
org = OrganizationFactory()
|
|
14
14
|
|
|
15
|
-
cli("badges", "toggle", str(org.id), PUBLIC_SERVICE)
|
|
15
|
+
self.cli("badges", "toggle", str(org.id), PUBLIC_SERVICE)
|
|
16
16
|
|
|
17
17
|
org.reload()
|
|
18
18
|
assert org.badges[0].kind == PUBLIC_SERVICE
|
|
19
19
|
|
|
20
|
-
def test_toggle_badge_off(self
|
|
20
|
+
def test_toggle_badge_off(self):
|
|
21
21
|
org = OrganizationFactory()
|
|
22
22
|
org.add_badge(PUBLIC_SERVICE)
|
|
23
23
|
org.add_badge(CERTIFIED)
|
|
24
24
|
|
|
25
|
-
cli("badges", "toggle", str(org.id), PUBLIC_SERVICE)
|
|
25
|
+
self.cli("badges", "toggle", str(org.id), PUBLIC_SERVICE)
|
|
26
26
|
|
|
27
27
|
org.reload()
|
|
28
28
|
assert org.badges[0].kind == CERTIFIED
|
|
29
29
|
|
|
30
|
-
def test_toggle_badge_on_from_file(self
|
|
30
|
+
def test_toggle_badge_on_from_file(self):
|
|
31
31
|
orgs = [OrganizationFactory() for _ in range(2)]
|
|
32
32
|
|
|
33
33
|
with NamedTemporaryFile(mode="w") as temp:
|
|
34
34
|
temp.write("\n".join((str(org.id) for org in orgs)))
|
|
35
35
|
temp.flush()
|
|
36
36
|
|
|
37
|
-
cli("badges", "toggle", temp.name, PUBLIC_SERVICE)
|
|
37
|
+
self.cli("badges", "toggle", temp.name, PUBLIC_SERVICE)
|
|
38
38
|
|
|
39
39
|
for org in orgs:
|
|
40
40
|
org.reload()
|
udata/core/csv.py
CHANGED
|
@@ -5,6 +5,7 @@ from datetime import date, datetime
|
|
|
5
5
|
from io import StringIO
|
|
6
6
|
|
|
7
7
|
from flask import Response, stream_with_context
|
|
8
|
+
from mongoengine.queryset import QuerySet
|
|
8
9
|
|
|
9
10
|
from udata.mongo import db
|
|
10
11
|
from udata.utils import recursive_get
|
|
@@ -35,6 +36,10 @@ class Adapter(object):
|
|
|
35
36
|
fields = None
|
|
36
37
|
|
|
37
38
|
def __init__(self, queryset):
|
|
39
|
+
# no_cache() to avoid eating up too much RAM when iterating over large querysets.
|
|
40
|
+
# Applied here rather than upstream to preserve custom QuerySet methods (like with_badge).
|
|
41
|
+
if isinstance(queryset, QuerySet):
|
|
42
|
+
queryset = queryset.no_cache()
|
|
38
43
|
self.queryset = queryset
|
|
39
44
|
self._fields = None
|
|
40
45
|
|
|
@@ -130,7 +130,7 @@ def filter_by_topic(base_query, filter_value):
|
|
|
130
130
|
try:
|
|
131
131
|
topic = Topic.objects.get(id=filter_value)
|
|
132
132
|
except Topic.DoesNotExist:
|
|
133
|
-
|
|
133
|
+
return base_query
|
|
134
134
|
else:
|
|
135
135
|
return base_query.filter(
|
|
136
136
|
id__in=[
|
|
@@ -140,11 +140,23 @@ def filter_by_topic(base_query, filter_value):
|
|
|
140
140
|
)
|
|
141
141
|
|
|
142
142
|
|
|
143
|
+
def filter_by_reuse(base_query, filter_value):
|
|
144
|
+
from udata.core.reuse.models import Reuse
|
|
145
|
+
|
|
146
|
+
try:
|
|
147
|
+
reuse = Reuse.objects.get(id=filter_value)
|
|
148
|
+
except Reuse.DoesNotExist:
|
|
149
|
+
return base_query
|
|
150
|
+
else:
|
|
151
|
+
return base_query.filter(id__in=[dataservice.id for dataservice in reuse.dataservices])
|
|
152
|
+
|
|
153
|
+
|
|
143
154
|
@generate_fields(
|
|
144
155
|
searchable=True,
|
|
145
156
|
nested_filters={"organization_badge": "organization.badges"},
|
|
146
157
|
standalone_filters=[
|
|
147
|
-
{"key": "topic", "constraints": "objectid", "query": filter_by_topic, "type": str}
|
|
158
|
+
{"key": "topic", "constraints": ["objectid"], "query": filter_by_topic, "type": str},
|
|
159
|
+
{"key": "reuse", "constraints": ["objectid"], "query": filter_by_reuse, "type": str},
|
|
148
160
|
],
|
|
149
161
|
additional_sorts=[
|
|
150
162
|
{"key": "followers", "value": "metrics.followers"},
|
|
@@ -297,7 +309,7 @@ class Dataservice(
|
|
|
297
309
|
|
|
298
310
|
@field(description="Link to the udata web page for this dataservice", show_as_ref=True)
|
|
299
311
|
def self_web_url(self, **kwargs):
|
|
300
|
-
return cdata_url(f"/dataservices/{self._link_id(**kwargs)}
|
|
312
|
+
return cdata_url(f"/dataservices/{self._link_id(**kwargs)}", **kwargs)
|
|
301
313
|
|
|
302
314
|
__metrics_keys__ = [
|
|
303
315
|
"discussions",
|
udata/core/dataservices/tasks.py
CHANGED
|
@@ -6,6 +6,7 @@ from udata.core.constants import HVD
|
|
|
6
6
|
from udata.core.dataservices.models import Dataservice
|
|
7
7
|
from udata.core.organization.constants import CERTIFIED, PUBLIC_SERVICE
|
|
8
8
|
from udata.core.organization.models import Organization
|
|
9
|
+
from udata.core.pages.models import Page
|
|
9
10
|
from udata.core.topic.models import TopicElement
|
|
10
11
|
from udata.harvest.models import HarvestJob
|
|
11
12
|
from udata.models import Discussion, Follow, Transfer
|
|
@@ -28,6 +29,12 @@ def purge_dataservices(self):
|
|
|
28
29
|
Transfer.objects(subject=dataservice).delete()
|
|
29
30
|
# Remove dataservices references in Topics
|
|
30
31
|
TopicElement.objects(element=dataservice).update(element=None)
|
|
32
|
+
# Remove dataservices in pages (mongoengine doesn't support updating a field in a generic embed)
|
|
33
|
+
Page._get_collection().update_many(
|
|
34
|
+
{"blocs.dataservices": dataservice.id},
|
|
35
|
+
{"$pull": {"blocs.$[b].dataservices": dataservice.id}},
|
|
36
|
+
array_filters=[{"b.dataservices": dataservice.id}],
|
|
37
|
+
)
|
|
31
38
|
# Remove dataservice
|
|
32
39
|
dataservice.delete()
|
|
33
40
|
|
udata/core/dataset/api.py
CHANGED
|
@@ -531,6 +531,8 @@ class ResourcesAPI(API):
|
|
|
531
531
|
f"All resources must be reordered, you provided {len(resources)} "
|
|
532
532
|
f"out of {len(dataset.resources)}",
|
|
533
533
|
)
|
|
534
|
+
if any(isinstance(r, dict) and "id" not in r for r in resources):
|
|
535
|
+
api.abort(400, "Each resource must have an 'id' field")
|
|
534
536
|
if set(r["id"] if isinstance(r, dict) else r for r in resources) != set(
|
|
535
537
|
str(r.id) for r in dataset.resources
|
|
536
538
|
):
|
udata/core/dataset/models.py
CHANGED
|
@@ -730,7 +730,7 @@ class Dataset(
|
|
|
730
730
|
}
|
|
731
731
|
|
|
732
732
|
def self_web_url(self, **kwargs):
|
|
733
|
-
return cdata_url(f"/datasets/{self._link_id(**kwargs)}
|
|
733
|
+
return cdata_url(f"/datasets/{self._link_id(**kwargs)}", **kwargs)
|
|
734
734
|
|
|
735
735
|
def self_api_url(self, **kwargs):
|
|
736
736
|
return url_for(
|
|
@@ -795,7 +795,7 @@ class Dataset(
|
|
|
795
795
|
Resources should be fetched when calling this method.
|
|
796
796
|
"""
|
|
797
797
|
if self.harvest and self.harvest.modified_at:
|
|
798
|
-
return self.harvest.modified_at
|
|
798
|
+
return to_naive_datetime(self.harvest.modified_at)
|
|
799
799
|
if self.resources:
|
|
800
800
|
return max([res.last_modified for res in self.resources])
|
|
801
801
|
else:
|
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
from flask_principal import Permission as BasePermission
|
|
2
|
+
from flask_principal import RoleNeed
|
|
3
|
+
|
|
1
4
|
from udata.auth import Permission, UserNeed
|
|
2
5
|
from udata.core.organization.permissions import (
|
|
3
6
|
OrganizationAdminNeed,
|
|
@@ -22,6 +25,34 @@ class OwnablePermission(Permission):
|
|
|
22
25
|
super(OwnablePermission, self).__init__(*needs)
|
|
23
26
|
|
|
24
27
|
|
|
28
|
+
class OwnableReadPermission(BasePermission):
|
|
29
|
+
"""Permission to read a private ownable object.
|
|
30
|
+
|
|
31
|
+
Always grants access if the object is not private.
|
|
32
|
+
For private objects, requires owner, org member, or sysadmin.
|
|
33
|
+
|
|
34
|
+
We inherit from BasePermission instead of udata's Permission because
|
|
35
|
+
Permission automatically adds RoleNeed("admin") to all needs. This means
|
|
36
|
+
a permission with no needs would only allow admins. With BasePermission,
|
|
37
|
+
an empty needs set allows everyone (Flask-Principal returns True when
|
|
38
|
+
self.needs is empty).
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
def __init__(self, obj):
|
|
42
|
+
if not getattr(obj, "private", False):
|
|
43
|
+
super().__init__()
|
|
44
|
+
return
|
|
45
|
+
|
|
46
|
+
needs = [RoleNeed("admin")]
|
|
47
|
+
if obj.organization:
|
|
48
|
+
needs.append(OrganizationAdminNeed(obj.organization.id))
|
|
49
|
+
needs.append(OrganizationEditorNeed(obj.organization.id))
|
|
50
|
+
elif obj.owner:
|
|
51
|
+
needs.append(UserNeed(obj.owner.fs_uniquifier))
|
|
52
|
+
|
|
53
|
+
super().__init__(*needs)
|
|
54
|
+
|
|
55
|
+
|
|
25
56
|
class DatasetEditPermission(OwnablePermission):
|
|
26
57
|
"""Permissions to edit a Dataset"""
|
|
27
58
|
|
udata/core/dataset/tasks.py
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import collections
|
|
2
|
+
import gzip
|
|
2
3
|
import os
|
|
3
|
-
from datetime import datetime
|
|
4
|
+
from datetime import date, datetime
|
|
4
5
|
from tempfile import NamedTemporaryFile
|
|
5
6
|
|
|
6
7
|
from celery.utils.log import get_task_logger
|
|
@@ -15,8 +16,10 @@ from udata.core.dataservices.models import Dataservice
|
|
|
15
16
|
from udata.core.dataset.constants import INSPIRE
|
|
16
17
|
from udata.core.organization.constants import CERTIFIED, PUBLIC_SERVICE
|
|
17
18
|
from udata.core.organization.models import Organization
|
|
19
|
+
from udata.core.pages.models import Page
|
|
18
20
|
from udata.harvest.models import HarvestJob
|
|
19
21
|
from udata.models import Activity, Discussion, Follow, TopicElement, Transfer, db
|
|
22
|
+
from udata.storage.s3 import store_bytes
|
|
20
23
|
from udata.tasks import job
|
|
21
24
|
|
|
22
25
|
from .models import Checksum, CommunityResource, Dataset, Resource
|
|
@@ -53,6 +56,12 @@ def purge_datasets(self):
|
|
|
53
56
|
dataservice.update(datasets=datasets)
|
|
54
57
|
# Remove HarvestItem references
|
|
55
58
|
HarvestJob.objects(items__dataset=dataset).update(set__items__S__dataset=None)
|
|
59
|
+
# Remove datasets in pages (mongoengine doesn't support updating a field in a generic embed)
|
|
60
|
+
Page._get_collection().update_many(
|
|
61
|
+
{"blocs.datasets": dataset.id},
|
|
62
|
+
{"$pull": {"blocs.$[b].datasets": dataset.id}},
|
|
63
|
+
array_filters=[{"b.datasets": dataset.id}],
|
|
64
|
+
)
|
|
56
65
|
# Remove associated Transfers
|
|
57
66
|
Transfer.objects(subject=dataset).delete()
|
|
58
67
|
# Remove each dataset's resource's file
|
|
@@ -86,16 +95,17 @@ def get_queryset(model_cls):
|
|
|
86
95
|
for attr in attrs:
|
|
87
96
|
if getattr(model_cls, attr, None):
|
|
88
97
|
params[attr] = False
|
|
89
|
-
|
|
90
|
-
|
|
98
|
+
return model_cls.objects.filter(**params)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def get_resource_for_csv_export_model(model, dataset):
|
|
102
|
+
for resource in dataset.resources:
|
|
103
|
+
if resource.extras.get("csv-export:model", "") == model:
|
|
104
|
+
return resource
|
|
91
105
|
|
|
92
106
|
|
|
93
107
|
def get_or_create_resource(r_info, model, dataset):
|
|
94
|
-
resource =
|
|
95
|
-
for r in dataset.resources:
|
|
96
|
-
if r.extras.get("csv-export:model", "") == model:
|
|
97
|
-
resource = r
|
|
98
|
-
break
|
|
108
|
+
resource = get_resource_for_csv_export_model(model, dataset)
|
|
99
109
|
if resource:
|
|
100
110
|
for k, v in r_info.items():
|
|
101
111
|
setattr(resource, k, v)
|
|
@@ -126,11 +136,16 @@ def store_resource(csvfile, model, dataset):
|
|
|
126
136
|
return get_or_create_resource(r_info, model, dataset)
|
|
127
137
|
|
|
128
138
|
|
|
129
|
-
def export_csv_for_model(model, dataset):
|
|
139
|
+
def export_csv_for_model(model, dataset, replace: bool = False):
|
|
130
140
|
model_cls = getattr(udata_models, model.capitalize(), None)
|
|
131
141
|
if not model_cls:
|
|
132
142
|
log.error("Unknow model %s" % model)
|
|
133
143
|
return
|
|
144
|
+
|
|
145
|
+
fs_filename_to_remove = None
|
|
146
|
+
if existing_resource := get_resource_for_csv_export_model(model, dataset):
|
|
147
|
+
fs_filename_to_remove = existing_resource.fs_filename
|
|
148
|
+
|
|
134
149
|
queryset = get_queryset(model_cls)
|
|
135
150
|
adapter = csv.get_adapter(model_cls)
|
|
136
151
|
if not adapter:
|
|
@@ -156,6 +171,15 @@ def export_csv_for_model(model, dataset):
|
|
|
156
171
|
else:
|
|
157
172
|
dataset.last_modified_internal = datetime.utcnow()
|
|
158
173
|
dataset.save()
|
|
174
|
+
# remove previous catalog if exists and replace is True
|
|
175
|
+
if replace and fs_filename_to_remove:
|
|
176
|
+
try:
|
|
177
|
+
storages.resources.delete(fs_filename_to_remove)
|
|
178
|
+
except FileNotFoundError:
|
|
179
|
+
log.error(
|
|
180
|
+
f"File not found while deleting resource #{resource.id} ({fs_filename_to_remove}) in export_csv_for_model cleanup"
|
|
181
|
+
)
|
|
182
|
+
return resource
|
|
159
183
|
finally:
|
|
160
184
|
csvfile.close()
|
|
161
185
|
os.unlink(csvfile.name)
|
|
@@ -184,7 +208,23 @@ def export_csv(self, model=None):
|
|
|
184
208
|
|
|
185
209
|
models = (model,) if model else ALLOWED_MODELS
|
|
186
210
|
for model in models:
|
|
187
|
-
export_csv_for_model(model, dataset)
|
|
211
|
+
resource = export_csv_for_model(model, dataset, replace=True)
|
|
212
|
+
|
|
213
|
+
# If we are the first day of the month, archive today catalogs
|
|
214
|
+
if (
|
|
215
|
+
current_app.config["EXPORT_CSV_ARCHIVE_S3_BUCKET"]
|
|
216
|
+
and resource
|
|
217
|
+
and date.today().day == 1
|
|
218
|
+
):
|
|
219
|
+
log.info(
|
|
220
|
+
f"Archiving {model} csv catalog on {current_app.config['EXPORT_CSV_ARCHIVE_S3_BUCKET']} bucket"
|
|
221
|
+
)
|
|
222
|
+
with storages.resources.open(resource.fs_filename, "rb") as f:
|
|
223
|
+
store_bytes(
|
|
224
|
+
bucket=current_app.config["EXPORT_CSV_ARCHIVE_S3_BUCKET"],
|
|
225
|
+
filename=f"{current_app.config['EXPORT_CSV_ARCHIVE_S3_FILENAME_PREFIX']}{resource.title}.gz",
|
|
226
|
+
bytes=gzip.compress(f.read()),
|
|
227
|
+
)
|
|
188
228
|
|
|
189
229
|
|
|
190
230
|
@job("bind-tabular-dataservice")
|
udata/core/discussions/models.py
CHANGED
|
@@ -14,6 +14,7 @@ log = logging.getLogger(__name__)
|
|
|
14
14
|
|
|
15
15
|
|
|
16
16
|
class Message(SpamMixin, db.EmbeddedDocument):
|
|
17
|
+
id = db.AutoUUIDField()
|
|
17
18
|
content = db.StringField(required=True)
|
|
18
19
|
posted_on = db.DateTimeField(default=datetime.utcnow, required=True)
|
|
19
20
|
posted_by = db.ReferenceField("User")
|
udata/core/metrics/__init__.py
CHANGED
|
@@ -1,6 +1,3 @@
|
|
|
1
|
-
from udata import entrypoints
|
|
2
|
-
|
|
3
|
-
|
|
4
1
|
def init_app(app):
|
|
5
2
|
# Load all core metrics
|
|
6
3
|
import udata.core.user.metrics # noqa
|
|
@@ -9,6 +6,3 @@ def init_app(app):
|
|
|
9
6
|
import udata.core.dataset.metrics # noqa
|
|
10
7
|
import udata.core.reuse.metrics # noqa
|
|
11
8
|
import udata.core.followers.metrics # noqa
|
|
12
|
-
|
|
13
|
-
# Load metrics from plugins
|
|
14
|
-
entrypoints.get_enabled("udata.metrics", app)
|
udata/core/organization/api.py
CHANGED
|
@@ -383,12 +383,13 @@ class MembershipRequestAPI(API):
|
|
|
383
383
|
|
|
384
384
|
form = api.validate(MembershipRequestForm, membership_request)
|
|
385
385
|
|
|
386
|
-
if
|
|
386
|
+
if membership_request:
|
|
387
|
+
form.populate_obj(membership_request)
|
|
388
|
+
org.save()
|
|
389
|
+
else:
|
|
387
390
|
membership_request = MembershipRequest()
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
form.populate_obj(membership_request)
|
|
391
|
-
org.save()
|
|
391
|
+
form.populate_obj(membership_request)
|
|
392
|
+
org.add_membership_request(membership_request)
|
|
392
393
|
|
|
393
394
|
notify_membership_request.delay(str(org.id), str(membership_request.id))
|
|
394
395
|
|
|
@@ -424,6 +425,7 @@ class MembershipAcceptAPI(MembershipAPI):
|
|
|
424
425
|
org.members.append(member)
|
|
425
426
|
org.count_members()
|
|
426
427
|
org.save()
|
|
428
|
+
MembershipRequest.after_handle.send(membership_request, org=org)
|
|
427
429
|
|
|
428
430
|
notify_membership_response.delay(str(org.id), str(membership_request.id))
|
|
429
431
|
|
|
@@ -446,6 +448,7 @@ class MembershipRefuseAPI(MembershipAPI):
|
|
|
446
448
|
membership_request.refusal_comment = form.comment.data
|
|
447
449
|
|
|
448
450
|
org.save()
|
|
451
|
+
MembershipRequest.after_handle.send(membership_request, org=org)
|
|
449
452
|
|
|
450
453
|
notify_membership_response.delay(str(org.id), str(membership_request.id))
|
|
451
454
|
|
udata/core/organization/mails.py
CHANGED
|
@@ -16,7 +16,7 @@ def new_membership_request(org: Organization, request: MembershipRequest) -> Mai
|
|
|
16
16
|
)
|
|
17
17
|
),
|
|
18
18
|
LabelledContent(_("Reason for the request:"), request.comment),
|
|
19
|
-
MailCTA(_("See the request"), cdata_url(f"/admin/organizations/{org.id}/members
|
|
19
|
+
MailCTA(_("See the request"), cdata_url(f"/admin/organizations/{org.id}/members")),
|
|
20
20
|
],
|
|
21
21
|
)
|
|
22
22
|
|
|
@@ -81,6 +81,9 @@ class MembershipRequest(db.EmbeddedDocument):
|
|
|
81
81
|
comment = db.StringField()
|
|
82
82
|
refusal_comment = db.StringField()
|
|
83
83
|
|
|
84
|
+
after_create = Signal()
|
|
85
|
+
after_handle = Signal()
|
|
86
|
+
|
|
84
87
|
@property
|
|
85
88
|
def status_label(self):
|
|
86
89
|
return MEMBERSHIP_STATUS[self.status]
|
|
@@ -198,7 +201,7 @@ class Organization(
|
|
|
198
201
|
cls.before_save.send(document)
|
|
199
202
|
|
|
200
203
|
def self_web_url(self, **kwargs):
|
|
201
|
-
return cdata_url(f"/organizations/{self._link_id(**kwargs)}
|
|
204
|
+
return cdata_url(f"/organizations/{self._link_id(**kwargs)}", **kwargs)
|
|
202
205
|
|
|
203
206
|
def self_api_url(self, **kwargs):
|
|
204
207
|
return url_for(
|
|
@@ -304,6 +307,11 @@ class Organization(
|
|
|
304
307
|
def views_count(self):
|
|
305
308
|
return self.metrics.get("views", 0)
|
|
306
309
|
|
|
310
|
+
def add_membership_request(self, membership_request):
|
|
311
|
+
self.requests.append(membership_request)
|
|
312
|
+
self.save()
|
|
313
|
+
MembershipRequest.after_create.send(membership_request, org=self)
|
|
314
|
+
|
|
307
315
|
def count_members(self):
|
|
308
316
|
self.metrics["members"] = len(self.members)
|
|
309
317
|
self.save(signal_kwargs={"ignores": ["post_save"]})
|