udata 14.0.0__py3-none-any.whl → 14.5.1.dev6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of udata might be problematic. Click here for more details.
- udata/api/__init__.py +2 -0
- udata/api_fields.py +35 -4
- udata/app.py +18 -20
- udata/auth/__init__.py +29 -6
- udata/auth/forms.py +2 -2
- udata/auth/views.py +13 -6
- udata/commands/dcat.py +1 -1
- udata/commands/serve.py +3 -11
- udata/commands/tests/test_fixtures.py +9 -9
- udata/core/access_type/api.py +1 -1
- udata/core/access_type/constants.py +12 -8
- udata/core/activity/api.py +5 -6
- udata/core/badges/tests/test_commands.py +6 -6
- udata/core/csv.py +5 -0
- udata/core/dataservices/api.py +8 -1
- udata/core/dataservices/apiv2.py +2 -5
- udata/core/dataservices/models.py +5 -2
- udata/core/dataservices/rdf.py +2 -1
- udata/core/dataservices/tasks.py +13 -2
- udata/core/dataset/api.py +10 -0
- udata/core/dataset/models.py +6 -6
- udata/core/dataset/permissions.py +31 -0
- udata/core/dataset/rdf.py +8 -2
- udata/core/dataset/tasks.py +23 -7
- udata/core/discussions/api.py +15 -1
- udata/core/discussions/models.py +6 -0
- udata/core/legal/__init__.py +0 -0
- udata/core/legal/mails.py +128 -0
- udata/core/organization/api.py +16 -5
- udata/core/organization/apiv2.py +2 -3
- udata/core/organization/mails.py +1 -1
- udata/core/organization/models.py +15 -2
- udata/core/organization/notifications.py +84 -0
- udata/core/organization/permissions.py +1 -1
- udata/core/organization/tasks.py +3 -0
- udata/core/pages/tests/test_api.py +32 -0
- udata/core/post/api.py +24 -69
- udata/core/post/models.py +84 -16
- udata/core/post/tests/test_api.py +24 -1
- udata/core/reports/api.py +18 -0
- udata/core/reports/models.py +42 -2
- udata/core/reuse/api.py +8 -0
- udata/core/reuse/apiv2.py +2 -5
- udata/core/reuse/models.py +1 -1
- udata/core/reuse/tasks.py +7 -0
- udata/core/spatial/forms.py +2 -2
- udata/core/topic/models.py +8 -2
- udata/core/user/api.py +10 -3
- udata/core/user/models.py +12 -2
- udata/features/notifications/api.py +7 -18
- udata/features/notifications/models.py +56 -0
- udata/features/notifications/tasks.py +25 -0
- udata/flask_mongoengine/engine.py +0 -4
- udata/flask_mongoengine/pagination.py +1 -1
- udata/frontend/markdown.py +2 -1
- udata/harvest/actions.py +21 -1
- udata/harvest/api.py +25 -8
- udata/harvest/backends/base.py +27 -1
- udata/harvest/backends/ckan/harvesters.py +11 -2
- udata/harvest/backends/dcat.py +4 -1
- udata/harvest/commands.py +33 -0
- udata/harvest/filters.py +17 -6
- udata/harvest/models.py +16 -0
- udata/harvest/permissions.py +27 -0
- udata/harvest/tests/ckan/test_ckan_backend.py +33 -0
- udata/harvest/tests/test_actions.py +58 -5
- udata/harvest/tests/test_api.py +276 -122
- udata/harvest/tests/test_base_backend.py +86 -1
- udata/harvest/tests/test_dcat_backend.py +81 -10
- udata/harvest/tests/test_filters.py +6 -0
- udata/i18n.py +1 -4
- udata/mail.py +19 -1
- udata/migrations/2025-10-31-create-membership-request-notifications.py +55 -0
- udata/migrations/2025-12-04-add-uuid-to-discussion-messages.py +28 -0
- udata/mongo/slug_fields.py +1 -1
- udata/rdf.py +58 -10
- udata/routing.py +2 -2
- udata/settings.py +11 -0
- udata/tasks.py +1 -0
- udata/templates/mail/message.html +5 -31
- udata/tests/__init__.py +27 -2
- udata/tests/api/__init__.py +108 -21
- udata/tests/api/test_activities_api.py +36 -0
- udata/tests/api/test_auth_api.py +121 -95
- udata/tests/api/test_base_api.py +7 -4
- udata/tests/api/test_datasets_api.py +50 -19
- udata/tests/api/test_organizations_api.py +192 -197
- udata/tests/api/test_reports_api.py +157 -0
- udata/tests/api/test_reuses_api.py +147 -147
- udata/tests/api/test_security_api.py +12 -12
- udata/tests/api/test_swagger.py +4 -4
- udata/tests/api/test_tags_api.py +8 -8
- udata/tests/api/test_user_api.py +1 -1
- udata/tests/apiv2/test_search.py +30 -0
- udata/tests/apiv2/test_swagger.py +4 -4
- udata/tests/cli/test_cli_base.py +8 -9
- udata/tests/dataservice/test_dataservice_tasks.py +29 -0
- udata/tests/dataset/test_dataset_commands.py +4 -4
- udata/tests/dataset/test_dataset_model.py +66 -26
- udata/tests/dataset/test_dataset_rdf.py +99 -5
- udata/tests/dataset/test_dataset_tasks.py +25 -0
- udata/tests/frontend/test_auth.py +58 -1
- udata/tests/frontend/test_csv.py +0 -3
- udata/tests/helpers.py +31 -27
- udata/tests/organization/test_notifications.py +67 -2
- udata/tests/plugin.py +6 -261
- udata/tests/search/test_search_integration.py +33 -0
- udata/tests/site/test_site_csv_exports.py +22 -10
- udata/tests/test_activity.py +9 -9
- udata/tests/test_api_fields.py +10 -0
- udata/tests/test_dcat_commands.py +2 -2
- udata/tests/test_discussions.py +5 -5
- udata/tests/test_legal_mails.py +359 -0
- udata/tests/test_migrations.py +21 -21
- udata/tests/test_notifications.py +15 -57
- udata/tests/test_notifications_task.py +43 -0
- udata/tests/test_owned.py +81 -1
- udata/tests/test_storages.py +25 -19
- udata/tests/test_topics.py +77 -61
- udata/tests/test_uris.py +33 -0
- udata/tests/workers/test_jobs_commands.py +23 -23
- udata/translations/ar/LC_MESSAGES/udata.mo +0 -0
- udata/translations/ar/LC_MESSAGES/udata.po +187 -108
- udata/translations/de/LC_MESSAGES/udata.mo +0 -0
- udata/translations/de/LC_MESSAGES/udata.po +187 -108
- udata/translations/es/LC_MESSAGES/udata.mo +0 -0
- udata/translations/es/LC_MESSAGES/udata.po +187 -108
- udata/translations/fr/LC_MESSAGES/udata.mo +0 -0
- udata/translations/fr/LC_MESSAGES/udata.po +188 -109
- udata/translations/it/LC_MESSAGES/udata.mo +0 -0
- udata/translations/it/LC_MESSAGES/udata.po +187 -108
- udata/translations/pt/LC_MESSAGES/udata.mo +0 -0
- udata/translations/pt/LC_MESSAGES/udata.po +187 -108
- udata/translations/sr/LC_MESSAGES/udata.mo +0 -0
- udata/translations/sr/LC_MESSAGES/udata.po +187 -108
- udata/translations/udata.pot +215 -106
- udata/uris.py +0 -2
- udata-14.5.1.dev6.dist-info/METADATA +109 -0
- {udata-14.0.0.dist-info → udata-14.5.1.dev6.dist-info}/RECORD +143 -140
- udata/core/post/forms.py +0 -30
- udata/flask_mongoengine/json.py +0 -38
- udata/templates/mail/base.html +0 -105
- udata/templates/mail/base.txt +0 -6
- udata/templates/mail/button.html +0 -3
- udata/templates/mail/layouts/1-column.html +0 -19
- udata/templates/mail/layouts/2-columns.html +0 -20
- udata/templates/mail/layouts/center-panel.html +0 -16
- udata-14.0.0.dist-info/METADATA +0 -132
- {udata-14.0.0.dist-info → udata-14.5.1.dev6.dist-info}/WHEEL +0 -0
- {udata-14.0.0.dist-info → udata-14.5.1.dev6.dist-info}/entry_points.txt +0 -0
- {udata-14.0.0.dist-info → udata-14.5.1.dev6.dist-info}/licenses/LICENSE +0 -0
- {udata-14.0.0.dist-info → udata-14.5.1.dev6.dist-info}/top_level.txt +0 -0
udata/core/dataservices/rdf.py
CHANGED
|
@@ -31,6 +31,7 @@ def dataservice_from_rdf(
|
|
|
31
31
|
node,
|
|
32
32
|
all_datasets: list[Dataset],
|
|
33
33
|
remote_url_prefix: str | None = None,
|
|
34
|
+
dryrun: bool = False,
|
|
34
35
|
) -> Dataservice:
|
|
35
36
|
"""
|
|
36
37
|
Create or update a dataservice from a RDF/DCAT graph
|
|
@@ -51,7 +52,7 @@ def dataservice_from_rdf(
|
|
|
51
52
|
dataservice.machine_documentation_url = url_from_rdf(d, DCAT.endpointDescription)
|
|
52
53
|
|
|
53
54
|
roles = [ # Imbricated list of contact points for each role
|
|
54
|
-
contact_points_from_rdf(d, rdf_entity, role, dataservice)
|
|
55
|
+
contact_points_from_rdf(d, rdf_entity, role, dataservice, dryrun=dryrun)
|
|
55
56
|
for rdf_entity, role in CONTACT_POINT_ENTITY_TO_ROLE.items()
|
|
56
57
|
]
|
|
57
58
|
dataservice.contact_points = [ # Flattened list of contact points
|
udata/core/dataservices/tasks.py
CHANGED
|
@@ -6,6 +6,7 @@ from udata.core.constants import HVD
|
|
|
6
6
|
from udata.core.dataservices.models import Dataservice
|
|
7
7
|
from udata.core.organization.constants import CERTIFIED, PUBLIC_SERVICE
|
|
8
8
|
from udata.core.organization.models import Organization
|
|
9
|
+
from udata.core.pages.models import Page
|
|
9
10
|
from udata.core.topic.models import TopicElement
|
|
10
11
|
from udata.harvest.models import HarvestJob
|
|
11
12
|
from udata.models import Discussion, Follow, Transfer
|
|
@@ -22,12 +23,22 @@ def purge_dataservices(self):
|
|
|
22
23
|
Follow.objects(following=dataservice).delete()
|
|
23
24
|
# Remove discussions
|
|
24
25
|
Discussion.objects(subject=dataservice).delete()
|
|
25
|
-
# Remove HarvestItem references
|
|
26
|
-
HarvestJob.
|
|
26
|
+
# Remove HarvestItem references (using update_many with array_filters to update all matching items)
|
|
27
|
+
HarvestJob._get_collection().update_many(
|
|
28
|
+
{"items.dataservice": dataservice.id},
|
|
29
|
+
{"$set": {"items.$[item].dataservice": None}},
|
|
30
|
+
array_filters=[{"item.dataservice": dataservice.id}],
|
|
31
|
+
)
|
|
27
32
|
# Remove associated Transfers
|
|
28
33
|
Transfer.objects(subject=dataservice).delete()
|
|
29
34
|
# Remove dataservices references in Topics
|
|
30
35
|
TopicElement.objects(element=dataservice).update(element=None)
|
|
36
|
+
# Remove dataservices in pages (mongoengine doesn't support updating a field in a generic embed)
|
|
37
|
+
Page._get_collection().update_many(
|
|
38
|
+
{"blocs.dataservices": dataservice.id},
|
|
39
|
+
{"$pull": {"blocs.$[b].dataservices": dataservice.id}},
|
|
40
|
+
array_filters=[{"b.dataservices": dataservice.id}],
|
|
41
|
+
)
|
|
31
42
|
# Remove dataservice
|
|
32
43
|
dataservice.delete()
|
|
33
44
|
|
udata/core/dataset/api.py
CHANGED
|
@@ -39,6 +39,7 @@ from udata.core.dataservices.models import Dataservice
|
|
|
39
39
|
from udata.core.dataset.models import CHECKSUM_TYPES
|
|
40
40
|
from udata.core.followers.api import FollowAPI
|
|
41
41
|
from udata.core.followers.models import Follow
|
|
42
|
+
from udata.core.legal.mails import add_send_legal_notice_argument, send_legal_notice_on_deletion
|
|
42
43
|
from udata.core.organization.models import Organization
|
|
43
44
|
from udata.core.reuse.models import Reuse
|
|
44
45
|
from udata.core.storages.api import handle_upload, upload_parser
|
|
@@ -364,6 +365,9 @@ class DatasetsAtomFeedAPI(API):
|
|
|
364
365
|
return response
|
|
365
366
|
|
|
366
367
|
|
|
368
|
+
dataset_delete_parser = add_send_legal_notice_argument(api.parser())
|
|
369
|
+
|
|
370
|
+
|
|
367
371
|
@ns.route("/<dataset:dataset>/", endpoint="dataset", doc=common_doc)
|
|
368
372
|
@api.response(404, "Dataset not found")
|
|
369
373
|
@api.response(410, "Dataset has been deleted")
|
|
@@ -397,12 +401,16 @@ class DatasetAPI(API):
|
|
|
397
401
|
|
|
398
402
|
@api.secure
|
|
399
403
|
@api.doc("delete_dataset")
|
|
404
|
+
@api.expect(dataset_delete_parser)
|
|
400
405
|
@api.response(204, "Dataset deleted")
|
|
401
406
|
def delete(self, dataset):
|
|
402
407
|
"""Delete a dataset given its identifier"""
|
|
408
|
+
args = dataset_delete_parser.parse_args()
|
|
403
409
|
if dataset.deleted:
|
|
404
410
|
api.abort(410, "Dataset has been deleted")
|
|
405
411
|
dataset.permissions["delete"].test()
|
|
412
|
+
send_legal_notice_on_deletion(dataset, args)
|
|
413
|
+
|
|
406
414
|
dataset.deleted = datetime.utcnow()
|
|
407
415
|
dataset.last_modified_internal = datetime.utcnow()
|
|
408
416
|
dataset.save()
|
|
@@ -531,6 +539,8 @@ class ResourcesAPI(API):
|
|
|
531
539
|
f"All resources must be reordered, you provided {len(resources)} "
|
|
532
540
|
f"out of {len(dataset.resources)}",
|
|
533
541
|
)
|
|
542
|
+
if any(isinstance(r, dict) and "id" not in r for r in resources):
|
|
543
|
+
api.abort(400, "Each resource must have an 'id' field")
|
|
534
544
|
if set(r["id"] if isinstance(r, dict) else r for r in resources) != set(
|
|
535
545
|
str(r.id) for r in dataset.resources
|
|
536
546
|
):
|
udata/core/dataset/models.py
CHANGED
|
@@ -546,7 +546,10 @@ class Dataset(
|
|
|
546
546
|
),
|
|
547
547
|
auditable=False,
|
|
548
548
|
)
|
|
549
|
-
description = field(
|
|
549
|
+
description = field(
|
|
550
|
+
db.StringField(required=True, default=""),
|
|
551
|
+
markdown=True,
|
|
552
|
+
)
|
|
550
553
|
description_short = field(db.StringField(max_length=DESCRIPTION_SHORT_SIZE_LIMIT))
|
|
551
554
|
license = field(db.ReferenceField("License"))
|
|
552
555
|
|
|
@@ -730,7 +733,7 @@ class Dataset(
|
|
|
730
733
|
}
|
|
731
734
|
|
|
732
735
|
def self_web_url(self, **kwargs):
|
|
733
|
-
return cdata_url(f"/datasets/{self._link_id(**kwargs)}
|
|
736
|
+
return cdata_url(f"/datasets/{self._link_id(**kwargs)}", **kwargs)
|
|
734
737
|
|
|
735
738
|
def self_api_url(self, **kwargs):
|
|
736
739
|
return url_for(
|
|
@@ -795,7 +798,7 @@ class Dataset(
|
|
|
795
798
|
Resources should be fetched when calling this method.
|
|
796
799
|
"""
|
|
797
800
|
if self.harvest and self.harvest.modified_at:
|
|
798
|
-
return self.harvest.modified_at
|
|
801
|
+
return to_naive_datetime(self.harvest.modified_at)
|
|
799
802
|
if self.resources:
|
|
800
803
|
return max([res.last_modified for res in self.resources])
|
|
801
804
|
else:
|
|
@@ -1148,9 +1151,6 @@ class ResourceSchema(object):
|
|
|
1148
1151
|
except requests.exceptions.RequestException as err:
|
|
1149
1152
|
log.exception(f"Error while getting schema catalog from {endpoint}: {err}")
|
|
1150
1153
|
schemas = cache.get(cache_key)
|
|
1151
|
-
except requests.exceptions.JSONDecodeError as err:
|
|
1152
|
-
log.exception(f"Error while getting schema catalog from {endpoint}: {err}")
|
|
1153
|
-
schemas = cache.get(cache_key)
|
|
1154
1154
|
else:
|
|
1155
1155
|
schemas = data.get("schemas", [])
|
|
1156
1156
|
cache.set(cache_key, schemas)
|
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
from flask_principal import Permission as BasePermission
|
|
2
|
+
from flask_principal import RoleNeed
|
|
3
|
+
|
|
1
4
|
from udata.auth import Permission, UserNeed
|
|
2
5
|
from udata.core.organization.permissions import (
|
|
3
6
|
OrganizationAdminNeed,
|
|
@@ -22,6 +25,34 @@ class OwnablePermission(Permission):
|
|
|
22
25
|
super(OwnablePermission, self).__init__(*needs)
|
|
23
26
|
|
|
24
27
|
|
|
28
|
+
class OwnableReadPermission(BasePermission):
|
|
29
|
+
"""Permission to read a private ownable object.
|
|
30
|
+
|
|
31
|
+
Always grants access if the object is not private.
|
|
32
|
+
For private objects, requires owner, org member, or sysadmin.
|
|
33
|
+
|
|
34
|
+
We inherit from BasePermission instead of udata's Permission because
|
|
35
|
+
Permission automatically adds RoleNeed("admin") to all needs. This means
|
|
36
|
+
a permission with no needs would only allow admins. With BasePermission,
|
|
37
|
+
an empty needs set allows everyone (Flask-Principal returns True when
|
|
38
|
+
self.needs is empty).
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
def __init__(self, obj):
|
|
42
|
+
if not getattr(obj, "private", False):
|
|
43
|
+
super().__init__()
|
|
44
|
+
return
|
|
45
|
+
|
|
46
|
+
needs = [RoleNeed("admin")]
|
|
47
|
+
if obj.organization:
|
|
48
|
+
needs.append(OrganizationAdminNeed(obj.organization.id))
|
|
49
|
+
needs.append(OrganizationEditorNeed(obj.organization.id))
|
|
50
|
+
elif obj.owner:
|
|
51
|
+
needs.append(UserNeed(obj.owner.fs_uniquifier))
|
|
52
|
+
|
|
53
|
+
super().__init__(*needs)
|
|
54
|
+
|
|
55
|
+
|
|
25
56
|
class DatasetEditPermission(OwnablePermission):
|
|
26
57
|
"""Permissions to edit a Dataset"""
|
|
27
58
|
|
udata/core/dataset/rdf.py
CHANGED
|
@@ -742,7 +742,13 @@ def resource_from_rdf(graph_or_distrib, dataset=None, is_additionnal=False):
|
|
|
742
742
|
return resource
|
|
743
743
|
|
|
744
744
|
|
|
745
|
-
def dataset_from_rdf(
|
|
745
|
+
def dataset_from_rdf(
|
|
746
|
+
graph: Graph,
|
|
747
|
+
dataset=None,
|
|
748
|
+
node=None,
|
|
749
|
+
remote_url_prefix: str | None = None,
|
|
750
|
+
dryrun: bool = False,
|
|
751
|
+
):
|
|
746
752
|
"""
|
|
747
753
|
Create or update a dataset from a RDF/DCAT graph
|
|
748
754
|
"""
|
|
@@ -764,7 +770,7 @@ def dataset_from_rdf(graph: Graph, dataset=None, node=None, remote_url_prefix: s
|
|
|
764
770
|
dataset.description = sanitize_html(description)
|
|
765
771
|
dataset.frequency = frequency_from_rdf(d.value(DCT.accrualPeriodicity)) or dataset.frequency
|
|
766
772
|
roles = [ # Imbricated list of contact points for each role
|
|
767
|
-
contact_points_from_rdf(d, rdf_entity, role, dataset)
|
|
773
|
+
contact_points_from_rdf(d, rdf_entity, role, dataset, dryrun=dryrun)
|
|
768
774
|
for rdf_entity, role in CONTACT_POINT_ENTITY_TO_ROLE.items()
|
|
769
775
|
]
|
|
770
776
|
dataset.contact_points = [ # Flattened list of contact points
|
udata/core/dataset/tasks.py
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import collections
|
|
2
|
+
import gzip
|
|
2
3
|
import os
|
|
3
4
|
from datetime import date, datetime
|
|
4
5
|
from tempfile import NamedTemporaryFile
|
|
@@ -15,6 +16,7 @@ from udata.core.dataservices.models import Dataservice
|
|
|
15
16
|
from udata.core.dataset.constants import INSPIRE
|
|
16
17
|
from udata.core.organization.constants import CERTIFIED, PUBLIC_SERVICE
|
|
17
18
|
from udata.core.organization.models import Organization
|
|
19
|
+
from udata.core.pages.models import Page
|
|
18
20
|
from udata.harvest.models import HarvestJob
|
|
19
21
|
from udata.models import Activity, Discussion, Follow, TopicElement, Transfer, db
|
|
20
22
|
from udata.storage.s3 import store_bytes
|
|
@@ -52,8 +54,18 @@ def purge_datasets(self):
|
|
|
52
54
|
datasets = dataservice.datasets
|
|
53
55
|
datasets.remove(dataset)
|
|
54
56
|
dataservice.update(datasets=datasets)
|
|
55
|
-
# Remove HarvestItem references
|
|
56
|
-
HarvestJob.
|
|
57
|
+
# Remove HarvestItem references (using update_many with array_filters to update all matching items)
|
|
58
|
+
HarvestJob._get_collection().update_many(
|
|
59
|
+
{"items.dataset": dataset.id},
|
|
60
|
+
{"$set": {"items.$[item].dataset": None}},
|
|
61
|
+
array_filters=[{"item.dataset": dataset.id}],
|
|
62
|
+
)
|
|
63
|
+
# Remove datasets in pages (mongoengine doesn't support updating a field in a generic embed)
|
|
64
|
+
Page._get_collection().update_many(
|
|
65
|
+
{"blocs.datasets": dataset.id},
|
|
66
|
+
{"$pull": {"blocs.$[b].datasets": dataset.id}},
|
|
67
|
+
array_filters=[{"b.datasets": dataset.id}],
|
|
68
|
+
)
|
|
57
69
|
# Remove associated Transfers
|
|
58
70
|
Transfer.objects(subject=dataset).delete()
|
|
59
71
|
# Remove each dataset's resource's file
|
|
@@ -87,8 +99,7 @@ def get_queryset(model_cls):
|
|
|
87
99
|
for attr in attrs:
|
|
88
100
|
if getattr(model_cls, attr, None):
|
|
89
101
|
params[attr] = False
|
|
90
|
-
|
|
91
|
-
return model_cls.objects.filter(**params).no_cache()
|
|
102
|
+
return model_cls.objects.filter(**params)
|
|
92
103
|
|
|
93
104
|
|
|
94
105
|
def get_resource_for_csv_export_model(model, dataset):
|
|
@@ -166,7 +177,12 @@ def export_csv_for_model(model, dataset, replace: bool = False):
|
|
|
166
177
|
dataset.save()
|
|
167
178
|
# remove previous catalog if exists and replace is True
|
|
168
179
|
if replace and fs_filename_to_remove:
|
|
169
|
-
|
|
180
|
+
try:
|
|
181
|
+
storages.resources.delete(fs_filename_to_remove)
|
|
182
|
+
except FileNotFoundError:
|
|
183
|
+
log.error(
|
|
184
|
+
f"File not found while deleting resource #{resource.id} ({fs_filename_to_remove}) in export_csv_for_model cleanup"
|
|
185
|
+
)
|
|
170
186
|
return resource
|
|
171
187
|
finally:
|
|
172
188
|
csvfile.close()
|
|
@@ -210,8 +226,8 @@ def export_csv(self, model=None):
|
|
|
210
226
|
with storages.resources.open(resource.fs_filename, "rb") as f:
|
|
211
227
|
store_bytes(
|
|
212
228
|
bucket=current_app.config["EXPORT_CSV_ARCHIVE_S3_BUCKET"],
|
|
213
|
-
filename=f"{current_app.config['EXPORT_CSV_ARCHIVE_S3_FILENAME_PREFIX']}{resource.title}",
|
|
214
|
-
bytes=f.read(),
|
|
229
|
+
filename=f"{current_app.config['EXPORT_CSV_ARCHIVE_S3_FILENAME_PREFIX']}{resource.title}.gz",
|
|
230
|
+
bytes=gzip.compress(f.read()),
|
|
215
231
|
)
|
|
216
232
|
|
|
217
233
|
|
udata/core/discussions/api.py
CHANGED
|
@@ -7,6 +7,7 @@ from flask_security import current_user
|
|
|
7
7
|
from udata.api import API, api, fields
|
|
8
8
|
from udata.core.dataservices.models import Dataservice
|
|
9
9
|
from udata.core.dataset.models import Dataset
|
|
10
|
+
from udata.core.legal.mails import add_send_legal_notice_argument, send_legal_notice_on_deletion
|
|
10
11
|
from udata.core.organization.api_fields import org_ref_fields
|
|
11
12
|
from udata.core.organization.models import Organization
|
|
12
13
|
from udata.core.reuse.models import Reuse
|
|
@@ -164,6 +165,9 @@ class DiscussionSpamAPI(SpamAPIMixin):
|
|
|
164
165
|
model = Discussion
|
|
165
166
|
|
|
166
167
|
|
|
168
|
+
discussion_delete_parser = add_send_legal_notice_argument(api.parser())
|
|
169
|
+
|
|
170
|
+
|
|
167
171
|
@ns.route("/<id>/", endpoint="discussion")
|
|
168
172
|
class DiscussionAPI(API):
|
|
169
173
|
"""
|
|
@@ -236,11 +240,14 @@ class DiscussionAPI(API):
|
|
|
236
240
|
return discussion
|
|
237
241
|
|
|
238
242
|
@api.doc("delete_discussion")
|
|
243
|
+
@api.expect(discussion_delete_parser)
|
|
239
244
|
@api.response(403, "Not allowed to delete this discussion")
|
|
240
245
|
def delete(self, id):
|
|
241
246
|
"""Delete a discussion given its ID"""
|
|
247
|
+
args = discussion_delete_parser.parse_args()
|
|
242
248
|
discussion = Discussion.objects.get_or_404(id=id_or_404(id))
|
|
243
249
|
discussion.permissions["delete"].test()
|
|
250
|
+
send_legal_notice_on_deletion(discussion, args)
|
|
244
251
|
|
|
245
252
|
discussion.delete()
|
|
246
253
|
on_discussion_deleted.send(discussion)
|
|
@@ -259,6 +266,9 @@ class DiscussionCommentSpamAPI(SpamAPIMixin):
|
|
|
259
266
|
return discussion, discussion.discussion[cidx]
|
|
260
267
|
|
|
261
268
|
|
|
269
|
+
message_delete_parser = add_send_legal_notice_argument(api.parser())
|
|
270
|
+
|
|
271
|
+
|
|
262
272
|
@ns.route("/<id>/comments/<int:cidx>/", endpoint="discussion_comment")
|
|
263
273
|
class DiscussionCommentAPI(API):
|
|
264
274
|
"""
|
|
@@ -286,16 +296,20 @@ class DiscussionCommentAPI(API):
|
|
|
286
296
|
return discussion
|
|
287
297
|
|
|
288
298
|
@api.doc("delete_discussion_comment")
|
|
299
|
+
@api.expect(message_delete_parser)
|
|
289
300
|
@api.response(403, "Not allowed to delete this comment")
|
|
290
301
|
def delete(self, id, cidx):
|
|
291
302
|
"""Delete a comment given its index"""
|
|
303
|
+
args = message_delete_parser.parse_args()
|
|
292
304
|
discussion = Discussion.objects.get_or_404(id=id_or_404(id))
|
|
293
305
|
if len(discussion.discussion) <= cidx:
|
|
294
306
|
api.abort(404, "Comment does not exist")
|
|
295
307
|
elif cidx == 0:
|
|
296
308
|
api.abort(400, "You cannot delete the first comment of a discussion")
|
|
297
309
|
|
|
298
|
-
discussion.discussion[cidx]
|
|
310
|
+
message = discussion.discussion[cidx]
|
|
311
|
+
message.permissions["delete"].test()
|
|
312
|
+
send_legal_notice_on_deletion(message, args)
|
|
299
313
|
|
|
300
314
|
discussion.discussion.pop(cidx)
|
|
301
315
|
discussion.save()
|
udata/core/discussions/models.py
CHANGED
|
@@ -6,6 +6,7 @@ from flask_login import current_user
|
|
|
6
6
|
|
|
7
7
|
from udata.core.linkable import Linkable
|
|
8
8
|
from udata.core.spam.models import SpamMixin, spam_protected
|
|
9
|
+
from udata.i18n import lazy_gettext as _
|
|
9
10
|
from udata.mongo import db
|
|
10
11
|
|
|
11
12
|
from .signals import on_discussion_closed, on_new_discussion, on_new_discussion_comment
|
|
@@ -14,6 +15,9 @@ log = logging.getLogger(__name__)
|
|
|
14
15
|
|
|
15
16
|
|
|
16
17
|
class Message(SpamMixin, db.EmbeddedDocument):
|
|
18
|
+
verbose_name = _("message")
|
|
19
|
+
|
|
20
|
+
id = db.AutoUUIDField()
|
|
17
21
|
content = db.StringField(required=True)
|
|
18
22
|
posted_on = db.DateTimeField(default=datetime.utcnow, required=True)
|
|
19
23
|
posted_by = db.ReferenceField("User")
|
|
@@ -69,6 +73,8 @@ class Message(SpamMixin, db.EmbeddedDocument):
|
|
|
69
73
|
|
|
70
74
|
|
|
71
75
|
class Discussion(SpamMixin, Linkable, db.Document):
|
|
76
|
+
verbose_name = _("discussion")
|
|
77
|
+
|
|
72
78
|
user = db.ReferenceField("User")
|
|
73
79
|
organization = db.ReferenceField("Organization")
|
|
74
80
|
|
|
File without changes
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
from flask import current_app
|
|
2
|
+
from flask_babel import LazyString
|
|
3
|
+
from flask_login import current_user
|
|
4
|
+
from flask_restx.inputs import boolean
|
|
5
|
+
|
|
6
|
+
from udata.core.dataservices.models import Dataservice
|
|
7
|
+
from udata.core.dataset.models import Dataset
|
|
8
|
+
from udata.core.discussions.models import Discussion, Message
|
|
9
|
+
from udata.core.organization.models import Organization
|
|
10
|
+
from udata.core.reuse.models import Reuse
|
|
11
|
+
from udata.core.user.models import User
|
|
12
|
+
from udata.i18n import lazy_gettext as _
|
|
13
|
+
from udata.mail import Link, MailMessage, ParagraphWithLinks
|
|
14
|
+
|
|
15
|
+
DeletableObject = Dataset | Reuse | Dataservice | Organization | User | Discussion | Message
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def add_send_legal_notice_argument(parser):
|
|
19
|
+
"""Add the send_legal_notice argument to a parser.
|
|
20
|
+
|
|
21
|
+
When send_legal_notice=true is passed by an admin, a formal legal notice email
|
|
22
|
+
is sent to the content owner. This email includes terms of use references and
|
|
23
|
+
information about how to contest the deletion (administrative appeal).
|
|
24
|
+
"""
|
|
25
|
+
parser.add_argument(
|
|
26
|
+
"send_legal_notice",
|
|
27
|
+
type=boolean,
|
|
28
|
+
default=False,
|
|
29
|
+
location="args",
|
|
30
|
+
help="Send formal legal notice with appeal information to owner (admin only)",
|
|
31
|
+
)
|
|
32
|
+
return parser
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _get_recipients_for_organization(org: Organization) -> list[User]:
|
|
36
|
+
return [m.user for m in org.by_role("admin")]
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _get_recipients_for_owned_object(obj: Dataset | Reuse | Dataservice) -> list[User]:
|
|
40
|
+
if obj.owner:
|
|
41
|
+
return [obj.owner]
|
|
42
|
+
elif obj.organization:
|
|
43
|
+
return _get_recipients_for_organization(obj.organization)
|
|
44
|
+
return []
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def send_legal_notice_on_deletion(obj: DeletableObject, args: dict):
|
|
48
|
+
"""Send a formal legal notice email when content is deleted by an admin.
|
|
49
|
+
|
|
50
|
+
The email is only sent if:
|
|
51
|
+
- send_legal_notice=true was passed in args
|
|
52
|
+
- The current user is a sysadmin
|
|
53
|
+
"""
|
|
54
|
+
if not args.get("send_legal_notice") or not current_user.sysadmin:
|
|
55
|
+
return
|
|
56
|
+
|
|
57
|
+
if isinstance(obj, Organization):
|
|
58
|
+
recipients = _get_recipients_for_organization(obj)
|
|
59
|
+
elif isinstance(obj, User):
|
|
60
|
+
recipients = [obj]
|
|
61
|
+
elif isinstance(obj, Discussion):
|
|
62
|
+
recipients = [obj.user] if obj.user else []
|
|
63
|
+
elif isinstance(obj, Message):
|
|
64
|
+
recipients = [obj.posted_by] if obj.posted_by else []
|
|
65
|
+
else:
|
|
66
|
+
recipients = _get_recipients_for_owned_object(obj)
|
|
67
|
+
|
|
68
|
+
if recipients:
|
|
69
|
+
_content_deleted(obj.verbose_name).send(recipients)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _content_deleted(content_type_label: LazyString) -> MailMessage:
|
|
73
|
+
admin = current_user._get_current_object()
|
|
74
|
+
terms_of_use_url = current_app.config.get("TERMS_OF_USE_URL")
|
|
75
|
+
terms_of_use_deletion_article = current_app.config.get("TERMS_OF_USE_DELETION_ARTICLE")
|
|
76
|
+
telerecours_url = current_app.config.get("TELERECOURS_URL")
|
|
77
|
+
|
|
78
|
+
if terms_of_use_url and terms_of_use_deletion_article:
|
|
79
|
+
terms_paragraph = ParagraphWithLinks(
|
|
80
|
+
_(
|
|
81
|
+
'Our %(terms_link)s specify in point %(article)s that the platform is not "intended '
|
|
82
|
+
"to disseminate advertising content, promotions of private interests, content contrary "
|
|
83
|
+
"to public order, illegal content, spam and any contribution violating the applicable "
|
|
84
|
+
"legal framework. The Editor reserves the right, without prior notice, to remove or "
|
|
85
|
+
"make inaccessible content published on the Platform that has no connection with its "
|
|
86
|
+
'Purpose. The Editor does not carry out "a priori" control over publications. As soon '
|
|
87
|
+
"as the Editor becomes aware of content contrary to these terms of use, it acts quickly "
|
|
88
|
+
'to remove or make it inaccessible".',
|
|
89
|
+
terms_link=Link(_("terms of use"), terms_of_use_url),
|
|
90
|
+
article=terms_of_use_deletion_article,
|
|
91
|
+
)
|
|
92
|
+
)
|
|
93
|
+
else:
|
|
94
|
+
terms_paragraph = _(
|
|
95
|
+
'The platform is not "intended to disseminate advertising content, promotions of '
|
|
96
|
+
"private interests, content contrary to public order, illegal content, spam and any "
|
|
97
|
+
"contribution violating the applicable legal framework. The Editor reserves the right, "
|
|
98
|
+
"without prior notice, to remove or make inaccessible content published on the Platform "
|
|
99
|
+
'that has no connection with its Purpose. The Editor does not carry out "a priori" '
|
|
100
|
+
"control over publications. As soon as the Editor becomes aware of content contrary to "
|
|
101
|
+
'these terms of use, it acts quickly to remove or make it inaccessible".'
|
|
102
|
+
)
|
|
103
|
+
|
|
104
|
+
if telerecours_url:
|
|
105
|
+
appeal_paragraph = ParagraphWithLinks(
|
|
106
|
+
_(
|
|
107
|
+
"You may contest this decision within two months of its notification by filing "
|
|
108
|
+
"an administrative appeal (recours gracieux ou hiérarchique). You may also bring "
|
|
109
|
+
'the matter before the administrative court via the "%(telerecours_link)s" application.',
|
|
110
|
+
telerecours_link=Link(_("Télérecours citoyens"), telerecours_url),
|
|
111
|
+
)
|
|
112
|
+
)
|
|
113
|
+
else:
|
|
114
|
+
appeal_paragraph = _("You may contest this decision by contacting us.")
|
|
115
|
+
|
|
116
|
+
paragraphs = [
|
|
117
|
+
_("Your %(content_type)s has been deleted.", content_type=content_type_label),
|
|
118
|
+
terms_paragraph,
|
|
119
|
+
appeal_paragraph,
|
|
120
|
+
_("Best regards,"),
|
|
121
|
+
admin.fullname,
|
|
122
|
+
_("%(site)s team member", site=current_app.config.get("SITE_TITLE", "data.gouv.fr")),
|
|
123
|
+
]
|
|
124
|
+
|
|
125
|
+
return MailMessage(
|
|
126
|
+
subject=_("Deletion of your %(content_type)s", content_type=content_type_label),
|
|
127
|
+
paragraphs=paragraphs,
|
|
128
|
+
)
|
udata/core/organization/api.py
CHANGED
|
@@ -21,6 +21,7 @@ from udata.core.discussions.api import discussion_fields
|
|
|
21
21
|
from udata.core.discussions.csv import DiscussionCsvAdapter
|
|
22
22
|
from udata.core.discussions.models import Discussion
|
|
23
23
|
from udata.core.followers.api import FollowAPI
|
|
24
|
+
from udata.core.legal.mails import add_send_legal_notice_argument, send_legal_notice_on_deletion
|
|
24
25
|
from udata.core.reuse.models import Reuse
|
|
25
26
|
from udata.core.storages.api import (
|
|
26
27
|
image_parser,
|
|
@@ -137,6 +138,9 @@ class OrganizationListAPI(API):
|
|
|
137
138
|
return organization, 201
|
|
138
139
|
|
|
139
140
|
|
|
141
|
+
org_delete_parser = add_send_legal_notice_argument(api.parser())
|
|
142
|
+
|
|
143
|
+
|
|
140
144
|
@ns.route("/<org:org>/", endpoint="organization", doc=common_doc)
|
|
141
145
|
@api.response(404, "Organization not found")
|
|
142
146
|
@api.response(410, "Organization has been deleted")
|
|
@@ -170,12 +174,16 @@ class OrganizationAPI(API):
|
|
|
170
174
|
|
|
171
175
|
@api.secure
|
|
172
176
|
@api.doc("delete_organization")
|
|
177
|
+
@api.expect(org_delete_parser)
|
|
173
178
|
@api.response(204, "Organization deleted")
|
|
174
179
|
def delete(self, org):
|
|
175
180
|
"""Delete a organization given its identifier"""
|
|
181
|
+
args = org_delete_parser.parse_args()
|
|
176
182
|
if org.deleted:
|
|
177
183
|
api.abort(410, "Organization has been deleted")
|
|
178
184
|
EditOrganizationPermission(org).test()
|
|
185
|
+
send_legal_notice_on_deletion(org, args)
|
|
186
|
+
|
|
179
187
|
org.deleted = datetime.utcnow()
|
|
180
188
|
org.save()
|
|
181
189
|
return "", 204
|
|
@@ -383,12 +391,13 @@ class MembershipRequestAPI(API):
|
|
|
383
391
|
|
|
384
392
|
form = api.validate(MembershipRequestForm, membership_request)
|
|
385
393
|
|
|
386
|
-
if
|
|
394
|
+
if membership_request:
|
|
395
|
+
form.populate_obj(membership_request)
|
|
396
|
+
org.save()
|
|
397
|
+
else:
|
|
387
398
|
membership_request = MembershipRequest()
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
form.populate_obj(membership_request)
|
|
391
|
-
org.save()
|
|
399
|
+
form.populate_obj(membership_request)
|
|
400
|
+
org.add_membership_request(membership_request)
|
|
392
401
|
|
|
393
402
|
notify_membership_request.delay(str(org.id), str(membership_request.id))
|
|
394
403
|
|
|
@@ -424,6 +433,7 @@ class MembershipAcceptAPI(MembershipAPI):
|
|
|
424
433
|
org.members.append(member)
|
|
425
434
|
org.count_members()
|
|
426
435
|
org.save()
|
|
436
|
+
MembershipRequest.after_handle.send(membership_request, org=org)
|
|
427
437
|
|
|
428
438
|
notify_membership_response.delay(str(org.id), str(membership_request.id))
|
|
429
439
|
|
|
@@ -446,6 +456,7 @@ class MembershipRefuseAPI(MembershipAPI):
|
|
|
446
456
|
membership_request.refusal_comment = form.comment.data
|
|
447
457
|
|
|
448
458
|
org.save()
|
|
459
|
+
MembershipRequest.after_handle.send(membership_request, org=org)
|
|
449
460
|
|
|
450
461
|
notify_membership_response.delay(str(org.id), str(membership_request.id))
|
|
451
462
|
|
udata/core/organization/apiv2.py
CHANGED
|
@@ -3,7 +3,6 @@ from flask import request
|
|
|
3
3
|
from udata import search
|
|
4
4
|
from udata.api import API, apiv2
|
|
5
5
|
from udata.core.contact_point.api_fields import contact_point_fields
|
|
6
|
-
from udata.utils import multi_to_dict
|
|
7
6
|
|
|
8
7
|
from .api_fields import member_fields, org_fields, org_page_fields
|
|
9
8
|
from .permissions import EditOrganizationPermission
|
|
@@ -30,8 +29,8 @@ class OrganizationSearchAPI(API):
|
|
|
30
29
|
@apiv2.marshal_with(org_page_fields)
|
|
31
30
|
def get(self):
|
|
32
31
|
"""Search all organizations"""
|
|
33
|
-
search_parser.parse_args()
|
|
34
|
-
return search.query(OrganizationSearch, **
|
|
32
|
+
args = search_parser.parse_args()
|
|
33
|
+
return search.query(OrganizationSearch, **args)
|
|
35
34
|
|
|
36
35
|
|
|
37
36
|
@ns.route("/<org:org>/extras/", endpoint="organization_extras")
|
udata/core/organization/mails.py
CHANGED
|
@@ -16,7 +16,7 @@ def new_membership_request(org: Organization, request: MembershipRequest) -> Mai
|
|
|
16
16
|
)
|
|
17
17
|
),
|
|
18
18
|
LabelledContent(_("Reason for the request:"), request.comment),
|
|
19
|
-
MailCTA(_("See the request"), cdata_url(f"/admin/organizations/{org.id}/members
|
|
19
|
+
MailCTA(_("See the request"), cdata_url(f"/admin/organizations/{org.id}/members")),
|
|
20
20
|
],
|
|
21
21
|
)
|
|
22
22
|
|
|
@@ -81,6 +81,9 @@ class MembershipRequest(db.EmbeddedDocument):
|
|
|
81
81
|
comment = db.StringField()
|
|
82
82
|
refusal_comment = db.StringField()
|
|
83
83
|
|
|
84
|
+
after_create = Signal()
|
|
85
|
+
after_handle = Signal()
|
|
86
|
+
|
|
84
87
|
@property
|
|
85
88
|
def status_label(self):
|
|
86
89
|
return MEMBERSHIP_STATUS[self.status]
|
|
@@ -123,7 +126,10 @@ class Organization(
|
|
|
123
126
|
db.SlugField(max_length=255, required=True, populate_from="name", update=True, follow=True),
|
|
124
127
|
auditable=False,
|
|
125
128
|
)
|
|
126
|
-
description = field(
|
|
129
|
+
description = field(
|
|
130
|
+
db.StringField(required=True),
|
|
131
|
+
markdown=True,
|
|
132
|
+
)
|
|
127
133
|
url = field(db.URLField())
|
|
128
134
|
image_url = field(db.StringField())
|
|
129
135
|
logo = field(
|
|
@@ -162,6 +168,8 @@ class Organization(
|
|
|
162
168
|
"auto_create_index_on_save": True,
|
|
163
169
|
}
|
|
164
170
|
|
|
171
|
+
verbose_name = _("organization")
|
|
172
|
+
|
|
165
173
|
def __str__(self):
|
|
166
174
|
return self.name or ""
|
|
167
175
|
|
|
@@ -198,7 +206,7 @@ class Organization(
|
|
|
198
206
|
cls.before_save.send(document)
|
|
199
207
|
|
|
200
208
|
def self_web_url(self, **kwargs):
|
|
201
|
-
return cdata_url(f"/organizations/{self._link_id(**kwargs)}
|
|
209
|
+
return cdata_url(f"/organizations/{self._link_id(**kwargs)}", **kwargs)
|
|
202
210
|
|
|
203
211
|
def self_api_url(self, **kwargs):
|
|
204
212
|
return url_for(
|
|
@@ -304,6 +312,11 @@ class Organization(
|
|
|
304
312
|
def views_count(self):
|
|
305
313
|
return self.metrics.get("views", 0)
|
|
306
314
|
|
|
315
|
+
def add_membership_request(self, membership_request):
|
|
316
|
+
self.requests.append(membership_request)
|
|
317
|
+
self.save()
|
|
318
|
+
MembershipRequest.after_create.send(membership_request, org=self)
|
|
319
|
+
|
|
307
320
|
def count_members(self):
|
|
308
321
|
self.metrics["members"] = len(self.members)
|
|
309
322
|
self.save(signal_kwargs={"ignores": ["post_save"]})
|