udata 10.1.2.dev34172__py2.py3-none-any.whl → 10.1.3__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of udata might be problematic. Click here for more details.
- udata/__init__.py +1 -1
- udata/commands/fixtures.py +1 -1
- udata/core/dataservices/constants.py +11 -0
- udata/core/dataservices/csv.py +3 -3
- udata/core/dataservices/models.py +27 -12
- udata/core/dataservices/rdf.py +5 -3
- udata/core/dataservices/search.py +13 -5
- udata/core/dataset/api.py +18 -3
- udata/core/dataset/forms.py +8 -4
- udata/core/dataset/models.py +6 -0
- udata/core/metrics/commands.py +20 -1
- udata/core/organization/api_fields.py +3 -1
- udata/core/user/api.py +8 -1
- udata/core/user/api_fields.py +5 -0
- udata/core/user/models.py +16 -11
- udata/core/user/tasks.py +81 -2
- udata/core/user/tests/test_user_model.py +29 -12
- udata/features/transfer/api.py +7 -4
- udata/harvest/actions.py +5 -0
- udata/harvest/backends/base.py +22 -2
- udata/harvest/models.py +19 -0
- udata/harvest/tests/test_actions.py +12 -0
- udata/harvest/tests/test_base_backend.py +74 -8
- udata/harvest/tests/test_dcat_backend.py +1 -1
- udata/migrations/2025-01-05-dataservices-fields-changes.py +136 -0
- udata/settings.py +5 -0
- udata/templates/mail/account_inactivity.html +29 -0
- udata/templates/mail/account_inactivity.txt +22 -0
- udata/templates/mail/inactive_account_deleted.html +5 -0
- udata/templates/mail/inactive_account_deleted.txt +6 -0
- udata/tests/api/test_dataservices_api.py +41 -2
- udata/tests/api/test_datasets_api.py +58 -0
- udata/tests/api/test_me_api.py +1 -1
- udata/tests/api/test_transfer_api.py +38 -0
- udata/tests/api/test_user_api.py +47 -8
- udata/tests/dataservice/test_csv_adapter.py +2 -0
- udata/tests/dataset/test_dataset_model.py +14 -0
- udata/tests/user/test_user_tasks.py +144 -0
- udata/translations/ar/LC_MESSAGES/udata.mo +0 -0
- udata/translations/ar/LC_MESSAGES/udata.po +88 -60
- udata/translations/de/LC_MESSAGES/udata.mo +0 -0
- udata/translations/de/LC_MESSAGES/udata.po +88 -60
- udata/translations/es/LC_MESSAGES/udata.mo +0 -0
- udata/translations/es/LC_MESSAGES/udata.po +88 -60
- udata/translations/fr/LC_MESSAGES/udata.mo +0 -0
- udata/translations/fr/LC_MESSAGES/udata.po +88 -60
- udata/translations/it/LC_MESSAGES/udata.mo +0 -0
- udata/translations/it/LC_MESSAGES/udata.po +88 -60
- udata/translations/pt/LC_MESSAGES/udata.mo +0 -0
- udata/translations/pt/LC_MESSAGES/udata.po +88 -60
- udata/translations/sr/LC_MESSAGES/udata.mo +0 -0
- udata/translations/sr/LC_MESSAGES/udata.po +88 -60
- udata/translations/udata.pot +83 -54
- {udata-10.1.2.dev34172.dist-info → udata-10.1.3.dist-info}/METADATA +15 -2
- {udata-10.1.2.dev34172.dist-info → udata-10.1.3.dist-info}/RECORD +59 -52
- {udata-10.1.2.dev34172.dist-info → udata-10.1.3.dist-info}/LICENSE +0 -0
- {udata-10.1.2.dev34172.dist-info → udata-10.1.3.dist-info}/WHEEL +0 -0
- {udata-10.1.2.dev34172.dist-info → udata-10.1.3.dist-info}/entry_points.txt +0 -0
- {udata-10.1.2.dev34172.dist-info → udata-10.1.3.dist-info}/top_level.txt +0 -0
udata/__init__.py
CHANGED
udata/commands/fixtures.py
CHANGED
|
@@ -39,7 +39,7 @@ COMMUNITY_RES_URL = "/api/1/datasets/community_resources"
|
|
|
39
39
|
DISCUSSION_URL = "/api/1/discussions"
|
|
40
40
|
|
|
41
41
|
|
|
42
|
-
DEFAULT_FIXTURE_FILE_TAG: str = "
|
|
42
|
+
DEFAULT_FIXTURE_FILE_TAG: str = "v6.0.0"
|
|
43
43
|
DEFAULT_FIXTURE_FILE: str = f"https://raw.githubusercontent.com/opendatateam/udata-fixtures/{DEFAULT_FIXTURE_FILE_TAG}/results.json" # noqa
|
|
44
44
|
|
|
45
45
|
DEFAULT_FIXTURES_RESULTS_FILENAME: str = "results.json"
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
DATASERVICE_FORMATS = ["REST", "WMS", "WSL"]
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
DATASERVICE_ACCESS_TYPE_OPEN = "open"
|
|
5
|
+
DATASERVICE_ACCESS_TYPE_OPEN_WITH_ACCOUNT = "open_with_account"
|
|
6
|
+
DATASERVICE_ACCESS_TYPE_RESTRICTED = "restricted"
|
|
7
|
+
DATASERVICE_ACCESS_TYPES = [
|
|
8
|
+
DATASERVICE_ACCESS_TYPE_OPEN,
|
|
9
|
+
DATASERVICE_ACCESS_TYPE_OPEN_WITH_ACCOUNT,
|
|
10
|
+
DATASERVICE_ACCESS_TYPE_RESTRICTED,
|
|
11
|
+
]
|
udata/core/dataservices/csv.py
CHANGED
|
@@ -13,13 +13,13 @@ class DataserviceCsvAdapter(csv.Adapter):
|
|
|
13
13
|
("url", lambda d: d.self_web_url()),
|
|
14
14
|
"description",
|
|
15
15
|
"base_api_url",
|
|
16
|
-
"
|
|
16
|
+
"machine_documentation_url",
|
|
17
|
+
"technical_documentation_url",
|
|
17
18
|
"business_documentation_url",
|
|
18
19
|
"authorization_request_url",
|
|
19
20
|
"availability",
|
|
20
21
|
"rate_limiting",
|
|
21
|
-
"
|
|
22
|
-
"has_token",
|
|
22
|
+
"access_type",
|
|
23
23
|
"license",
|
|
24
24
|
("organization", "organization.name"),
|
|
25
25
|
("organization_id", "organization.id"),
|
|
@@ -8,6 +8,7 @@ from mongoengine.signals import post_save
|
|
|
8
8
|
import udata.core.contact_point.api_fields as contact_api_fields
|
|
9
9
|
import udata.core.dataset.api_fields as datasets_api_fields
|
|
10
10
|
from udata.api_fields import field, function_field, generate_fields
|
|
11
|
+
from udata.core.dataservices.constants import DATASERVICE_ACCESS_TYPES, DATASERVICE_FORMATS
|
|
11
12
|
from udata.core.dataset.models import Dataset
|
|
12
13
|
from udata.core.metrics.models import WithMetrics
|
|
13
14
|
from udata.core.owned import Owned, OwnedQuerySet
|
|
@@ -24,8 +25,6 @@ from udata.uris import endpoint_for
|
|
|
24
25
|
# "spatial"
|
|
25
26
|
# "temporal_coverage"
|
|
26
27
|
|
|
27
|
-
DATASERVICE_FORMATS = ["REST", "WMS", "WSL"]
|
|
28
|
-
|
|
29
28
|
|
|
30
29
|
class DataserviceQuerySet(OwnedQuerySet):
|
|
31
30
|
def visible(self):
|
|
@@ -95,16 +94,23 @@ class HarvestMetadata(db.EmbeddedDocument):
|
|
|
95
94
|
)
|
|
96
95
|
last_update = field(db.DateTimeField(), description="Date of the last harvesting")
|
|
97
96
|
archived_at = field(db.DateTimeField())
|
|
97
|
+
archived_reason = field(db.StringField())
|
|
98
98
|
|
|
99
99
|
|
|
100
100
|
@generate_fields(
|
|
101
101
|
searchable=True,
|
|
102
102
|
additional_filters={"organization_badge": "organization.badges"},
|
|
103
|
+
additional_sorts=[
|
|
104
|
+
{"key": "followers", "value": "metrics.followers"},
|
|
105
|
+
{"key": "views", "value": "metrics.views"},
|
|
106
|
+
],
|
|
103
107
|
)
|
|
104
108
|
class Dataservice(WithMetrics, Owned, db.Document):
|
|
105
109
|
meta = {
|
|
106
110
|
"indexes": [
|
|
107
111
|
"$title",
|
|
112
|
+
"metrics.followers",
|
|
113
|
+
"metrics.views",
|
|
108
114
|
]
|
|
109
115
|
+ Owned.meta["indexes"],
|
|
110
116
|
"queryset_class": DataserviceQuerySet,
|
|
@@ -132,13 +138,22 @@ class Dataservice(WithMetrics, Owned, db.Document):
|
|
|
132
138
|
)
|
|
133
139
|
description = field(db.StringField(default=""), description="In markdown")
|
|
134
140
|
base_api_url = field(db.URLField(), sortable=True)
|
|
135
|
-
|
|
141
|
+
|
|
142
|
+
machine_documentation_url = field(
|
|
143
|
+
db.URLField(), description="Swagger link, OpenAPI format, WMS XML…"
|
|
144
|
+
)
|
|
145
|
+
technical_documentation_url = field(db.URLField(), description="HTML version of a Swagger…")
|
|
136
146
|
business_documentation_url = field(db.URLField())
|
|
137
|
-
|
|
138
|
-
availability = field(db.FloatField(min=0, max=100), example="99.99")
|
|
147
|
+
|
|
139
148
|
rate_limiting = field(db.StringField())
|
|
140
|
-
|
|
141
|
-
|
|
149
|
+
rate_limiting_url = field(db.URLField())
|
|
150
|
+
|
|
151
|
+
availability = field(db.FloatField(min=0, max=100), example="99.99")
|
|
152
|
+
availability_url = field(db.URLField())
|
|
153
|
+
|
|
154
|
+
access_type = field(db.StringField(choices=DATASERVICE_ACCESS_TYPES), filterable={})
|
|
155
|
+
authorization_request_url = field(db.URLField())
|
|
156
|
+
|
|
142
157
|
format = field(db.StringField(choices=DATASERVICE_FORMATS))
|
|
143
158
|
|
|
144
159
|
license = field(
|
|
@@ -223,11 +238,11 @@ class Dataservice(WithMetrics, Owned, db.Document):
|
|
|
223
238
|
def self_web_url(self):
|
|
224
239
|
return endpoint_for("dataservices.show", dataservice=self, _external=True)
|
|
225
240
|
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
241
|
+
__metrics_keys__ = [
|
|
242
|
+
"discussions",
|
|
243
|
+
"followers",
|
|
244
|
+
"views",
|
|
245
|
+
]
|
|
231
246
|
|
|
232
247
|
@property
|
|
233
248
|
def is_hidden(self):
|
udata/core/dataservices/rdf.py
CHANGED
|
@@ -42,7 +42,9 @@ def dataservice_from_rdf(
|
|
|
42
42
|
dataservice.description = sanitize_html(d.value(DCT.description) or d.value(DCT.abstract))
|
|
43
43
|
|
|
44
44
|
dataservice.base_api_url = url_from_rdf(d, DCAT.endpointURL)
|
|
45
|
-
|
|
45
|
+
|
|
46
|
+
# TODO detect if it's human-readable or not?
|
|
47
|
+
dataservice.machine_documentation_url = url_from_rdf(d, DCAT.endpointDescription)
|
|
46
48
|
|
|
47
49
|
roles = [ # Imbricated list of contact points for each role
|
|
48
50
|
contact_points_from_rdf(d, rdf_entity, role, dataservice)
|
|
@@ -145,8 +147,8 @@ def dataservice_to_rdf(dataservice: Dataservice, graph=None):
|
|
|
145
147
|
),
|
|
146
148
|
)
|
|
147
149
|
|
|
148
|
-
if dataservice.
|
|
149
|
-
d.set(DCAT.endpointDescription, URIRef(dataservice.
|
|
150
|
+
if dataservice.machine_documentation_url:
|
|
151
|
+
d.set(DCAT.endpointDescription, URIRef(dataservice.machine_documentation_url))
|
|
150
152
|
|
|
151
153
|
# Add DCAT-AP HVD properties if the dataservice is tagged hvd.
|
|
152
154
|
# See https://semiceu.github.io/DCAT-AP/releases/2.2.0-hvd/
|
|
@@ -5,6 +5,11 @@ from flask_restx.inputs import boolean
|
|
|
5
5
|
|
|
6
6
|
from udata.api import api
|
|
7
7
|
from udata.api.parsers import ModelApiParser
|
|
8
|
+
from udata.core.dataservices.constants import (
|
|
9
|
+
DATASERVICE_ACCESS_TYPE_OPEN,
|
|
10
|
+
DATASERVICE_ACCESS_TYPE_OPEN_WITH_ACCOUNT,
|
|
11
|
+
DATASERVICE_ACCESS_TYPE_RESTRICTED,
|
|
12
|
+
)
|
|
8
13
|
from udata.models import Dataservice, Organization, User
|
|
9
14
|
from udata.search import (
|
|
10
15
|
BoolFilter,
|
|
@@ -47,7 +52,11 @@ class DataserviceApiParser(ModelApiParser):
|
|
|
47
52
|
api.abort(400, "Organization arg must be an identifier")
|
|
48
53
|
dataservices = dataservices.filter(organization=args["organization"])
|
|
49
54
|
if "is_restricted" in args:
|
|
50
|
-
dataservices = dataservices.filter(
|
|
55
|
+
dataservices = dataservices.filter(
|
|
56
|
+
access_type__in=[DATASERVICE_ACCESS_TYPE_RESTRICTED]
|
|
57
|
+
if boolean(args["is_restricted"])
|
|
58
|
+
else [DATASERVICE_ACCESS_TYPE_OPEN, DATASERVICE_ACCESS_TYPE_OPEN_WITH_ACCOUNT]
|
|
59
|
+
)
|
|
51
60
|
return dataservices
|
|
52
61
|
|
|
53
62
|
|
|
@@ -56,9 +65,7 @@ class DataserviceSearch(ModelSearchAdapter):
|
|
|
56
65
|
model = Dataservice
|
|
57
66
|
search_url = "dataservices/"
|
|
58
67
|
|
|
59
|
-
sorts = {
|
|
60
|
-
"created": "created_at",
|
|
61
|
-
}
|
|
68
|
+
sorts = {"created": "created_at", "views": "views", "followers": "followers"}
|
|
62
69
|
|
|
63
70
|
filters = {
|
|
64
71
|
"tag": Filter(),
|
|
@@ -114,5 +121,6 @@ class DataserviceSearch(ModelSearchAdapter):
|
|
|
114
121
|
"tags": dataservice.tags,
|
|
115
122
|
"extras": extras,
|
|
116
123
|
"followers": dataservice.metrics.get("followers", 0),
|
|
117
|
-
"is_restricted": dataservice.
|
|
124
|
+
"is_restricted": dataservice.access_type == DATASERVICE_ACCESS_TYPE_RESTRICTED,
|
|
125
|
+
"views": dataservice.metrics.get("views", 0),
|
|
118
126
|
}
|
udata/core/dataset/api.py
CHANGED
|
@@ -62,7 +62,12 @@ from .exceptions import (
|
|
|
62
62
|
SchemasCacheUnavailableException,
|
|
63
63
|
SchemasCatalogNotFoundException,
|
|
64
64
|
)
|
|
65
|
-
from .forms import
|
|
65
|
+
from .forms import (
|
|
66
|
+
CommunityResourceForm,
|
|
67
|
+
DatasetForm,
|
|
68
|
+
ResourceFormWithoutId,
|
|
69
|
+
ResourcesListForm,
|
|
70
|
+
)
|
|
66
71
|
from .models import (
|
|
67
72
|
Checksum,
|
|
68
73
|
CommunityResource,
|
|
@@ -379,8 +384,9 @@ class ResourcesAPI(API):
|
|
|
379
384
|
def post(self, dataset):
|
|
380
385
|
"""Create a new resource for a given dataset"""
|
|
381
386
|
ResourceEditPermission(dataset).test()
|
|
382
|
-
form = api.validate(
|
|
387
|
+
form = api.validate(ResourceFormWithoutId)
|
|
383
388
|
resource = Resource()
|
|
389
|
+
|
|
384
390
|
if form._fields.get("filetype").data != "remote":
|
|
385
391
|
api.abort(400, "This endpoint only supports remote resources")
|
|
386
392
|
form.populate_obj(resource)
|
|
@@ -545,10 +551,19 @@ class ResourceAPI(ResourceMixin, API):
|
|
|
545
551
|
"""Update a given resource on a given dataset"""
|
|
546
552
|
ResourceEditPermission(dataset).test()
|
|
547
553
|
resource = self.get_resource_or_404(dataset, rid)
|
|
548
|
-
form = api.validate(
|
|
554
|
+
form = api.validate(ResourceFormWithoutId, resource)
|
|
555
|
+
|
|
556
|
+
# ensure filetype is not modified after creation
|
|
557
|
+
if (
|
|
558
|
+
form._fields.get("filetype").data
|
|
559
|
+
and form._fields.get("filetype").data != resource.filetype
|
|
560
|
+
):
|
|
561
|
+
abort(400, "Cannot modify filetype after creation")
|
|
562
|
+
|
|
549
563
|
# ensure API client does not override url on self-hosted resources
|
|
550
564
|
if resource.filetype == "file":
|
|
551
565
|
form._fields.get("url").data = resource.url
|
|
566
|
+
|
|
552
567
|
# populate_obj populates existing resource object with the content of the form.
|
|
553
568
|
# update_resource saves the updated resource dict to the database
|
|
554
569
|
# the additional dataset.save is required as we update the last_modified date.
|
udata/core/dataset/forms.py
CHANGED
|
@@ -72,7 +72,7 @@ class BaseResourceForm(ModelForm):
|
|
|
72
72
|
[validators.DataRequired()],
|
|
73
73
|
choices=list(RESOURCE_FILETYPES.items()),
|
|
74
74
|
default="file",
|
|
75
|
-
description=_("Whether the resource is an uploaded file,
|
|
75
|
+
description=_("Whether the resource is an uploaded file, a remote file or an API"),
|
|
76
76
|
)
|
|
77
77
|
type = fields.RadioField(
|
|
78
78
|
_("Type"),
|
|
@@ -89,7 +89,7 @@ class BaseResourceForm(ModelForm):
|
|
|
89
89
|
checksum = fields.FormField(ChecksumForm)
|
|
90
90
|
mime = fields.StringField(
|
|
91
91
|
_("Mime type"),
|
|
92
|
-
description=_("The mime type associated to the extension.
|
|
92
|
+
description=_("The mime type associated to the extension. (ex: text/plain)"),
|
|
93
93
|
)
|
|
94
94
|
filesize = fields.IntegerField(
|
|
95
95
|
_("Size"), [validators.optional()], description=_("The file size in bytes")
|
|
@@ -104,6 +104,10 @@ class ResourceForm(BaseResourceForm):
|
|
|
104
104
|
id = fields.UUIDField()
|
|
105
105
|
|
|
106
106
|
|
|
107
|
+
class ResourceFormWithoutId(BaseResourceForm):
|
|
108
|
+
model_class = Resource
|
|
109
|
+
|
|
110
|
+
|
|
107
111
|
class CommunityResourceForm(BaseResourceForm):
|
|
108
112
|
model_class = CommunityResource
|
|
109
113
|
|
|
@@ -145,7 +149,7 @@ class DatasetForm(ModelForm):
|
|
|
145
149
|
description = fields.MarkdownField(
|
|
146
150
|
_("Description"),
|
|
147
151
|
[validators.DataRequired(), validators.Length(max=DESCRIPTION_SIZE_LIMIT)],
|
|
148
|
-
description=_("The details about the dataset
|
|
152
|
+
description=_("The details about the dataset (collection process, specifics...)."),
|
|
149
153
|
)
|
|
150
154
|
license = fields.ModelSelectField(_("License"), model=License, allow_blank=True)
|
|
151
155
|
frequency = fields.SelectField(
|
|
@@ -168,7 +172,7 @@ class DatasetForm(ModelForm):
|
|
|
168
172
|
tags = fields.TagField(_("Tags"), description=_("Some taxonomy keywords"))
|
|
169
173
|
private = fields.BooleanField(
|
|
170
174
|
_("Private"),
|
|
171
|
-
description=_("Restrict the dataset visibility to you or
|
|
175
|
+
description=_("Restrict the dataset visibility to you or your organization only."),
|
|
172
176
|
)
|
|
173
177
|
|
|
174
178
|
owner = fields.CurrentUserField()
|
udata/core/dataset/models.py
CHANGED
|
@@ -638,6 +638,9 @@ class Dataset(WithMetrics, DatasetBadgeMixin, Owned, db.Document):
|
|
|
638
638
|
if self.frequency in LEGACY_FREQUENCIES:
|
|
639
639
|
self.frequency = LEGACY_FREQUENCIES[self.frequency]
|
|
640
640
|
|
|
641
|
+
if len(set(res.id for res in self.resources)) != len(self.resources):
|
|
642
|
+
raise MongoEngineValidationError(f"Duplicate resource ID in dataset #{self.id}.")
|
|
643
|
+
|
|
641
644
|
for key, value in self.extras.items():
|
|
642
645
|
if not key.startswith("custom:"):
|
|
643
646
|
continue
|
|
@@ -897,6 +900,9 @@ class Dataset(WithMetrics, DatasetBadgeMixin, Owned, db.Document):
|
|
|
897
900
|
def add_resource(self, resource):
|
|
898
901
|
"""Perform an atomic prepend for a new resource"""
|
|
899
902
|
resource.validate()
|
|
903
|
+
if resource.id in [r.id for r in self.resources]:
|
|
904
|
+
raise MongoEngineValidationError("Cannot add resource with already existing ID")
|
|
905
|
+
|
|
900
906
|
self.update(
|
|
901
907
|
__raw__={"$push": {"resources": {"$each": [resource.to_mongo()], "$position": 0}}}
|
|
902
908
|
)
|
udata/core/metrics/commands.py
CHANGED
|
@@ -4,6 +4,7 @@ import click
|
|
|
4
4
|
from flask import current_app
|
|
5
5
|
|
|
6
6
|
from udata.commands import cli, success
|
|
7
|
+
from udata.core.dataservices.models import Dataservice
|
|
7
8
|
from udata.models import Dataset, GeoZone, Organization, Reuse, Site, User
|
|
8
9
|
|
|
9
10
|
log = logging.getLogger(__name__)
|
|
@@ -19,6 +20,7 @@ def grp():
|
|
|
19
20
|
@click.option("-s", "--site", is_flag=True, help="Update site metrics")
|
|
20
21
|
@click.option("-o", "--organizations", is_flag=True, help="Compute organizations metrics")
|
|
21
22
|
@click.option("-d", "--datasets", is_flag=True, help="Compute datasets metrics")
|
|
23
|
+
@click.option("--dataservices", is_flag=True, help="Compute dataservices metrics")
|
|
22
24
|
@click.option("-r", "--reuses", is_flag=True, help="Compute reuses metrics")
|
|
23
25
|
@click.option("-u", "--users", is_flag=True, help="Compute users metrics")
|
|
24
26
|
@click.option("-g", "--geozones", is_flag=True, help="Compute geo levels metrics")
|
|
@@ -28,12 +30,13 @@ def update(
|
|
|
28
30
|
organizations=False,
|
|
29
31
|
users=False,
|
|
30
32
|
datasets=False,
|
|
33
|
+
dataservices=False,
|
|
31
34
|
reuses=False,
|
|
32
35
|
geozones=False,
|
|
33
36
|
drop=False,
|
|
34
37
|
):
|
|
35
38
|
"""Update all metrics for the current date"""
|
|
36
|
-
do_all = not any((site, organizations, users, datasets, reuses, geozones))
|
|
39
|
+
do_all = not any((site, organizations, users, datasets, dataservices, reuses, geozones))
|
|
37
40
|
|
|
38
41
|
if do_all or site:
|
|
39
42
|
log.info("Update site metrics")
|
|
@@ -75,6 +78,22 @@ def update(
|
|
|
75
78
|
log.info(f"Error during update: {e}")
|
|
76
79
|
continue
|
|
77
80
|
|
|
81
|
+
if do_all or dataservices:
|
|
82
|
+
log.info("Update dataservices metrics")
|
|
83
|
+
all_dataservices = Dataservice.objects.visible().timeout(False)
|
|
84
|
+
with click.progressbar(
|
|
85
|
+
all_dataservices, length=Dataservice.objects.count()
|
|
86
|
+
) as dataservice_bar:
|
|
87
|
+
for dataservice in dataservice_bar:
|
|
88
|
+
try:
|
|
89
|
+
if drop:
|
|
90
|
+
dataservice.metrics.clear()
|
|
91
|
+
dataservice.count_discussions()
|
|
92
|
+
dataservice.count_followers()
|
|
93
|
+
except Exception as e:
|
|
94
|
+
log.info(f"Error during update: {e}")
|
|
95
|
+
continue
|
|
96
|
+
|
|
78
97
|
if do_all or reuses:
|
|
79
98
|
log.info("Update reuses metrics")
|
|
80
99
|
all_reuses = Reuse.objects.visible().timeout(False)
|
|
@@ -82,7 +82,9 @@ member_user_with_email_fields = api.inherit(
|
|
|
82
82
|
readonly=True,
|
|
83
83
|
),
|
|
84
84
|
"last_login_at": fields.Raw(
|
|
85
|
-
attribute=lambda o: o.
|
|
85
|
+
attribute=lambda o: o.current_login_at
|
|
86
|
+
if check_can_access_user_private_info()
|
|
87
|
+
else None,
|
|
86
88
|
description="The user last connection date (only present on show organization endpoint if the current user is member of the organization: admin or editor)",
|
|
87
89
|
readonly=True,
|
|
88
90
|
),
|
udata/core/user/api.py
CHANGED
|
@@ -275,6 +275,13 @@ delete_parser.add_argument(
|
|
|
275
275
|
location="args",
|
|
276
276
|
default=False,
|
|
277
277
|
)
|
|
278
|
+
delete_parser.add_argument(
|
|
279
|
+
"delete_comments",
|
|
280
|
+
type=bool,
|
|
281
|
+
help="Delete comments posted by the user upon user deletion",
|
|
282
|
+
location="args",
|
|
283
|
+
default=False,
|
|
284
|
+
)
|
|
278
285
|
|
|
279
286
|
|
|
280
287
|
@ns.route("/<user:user>/", endpoint="user")
|
|
@@ -317,7 +324,7 @@ class UserAPI(API):
|
|
|
317
324
|
403, "You cannot delete yourself with this API. " + 'Use the "me" API instead.'
|
|
318
325
|
)
|
|
319
326
|
|
|
320
|
-
user.mark_as_deleted(notify=not args["no_mail"])
|
|
327
|
+
user.mark_as_deleted(notify=not args["no_mail"], delete_comments=args["delete_comments"])
|
|
321
328
|
return "", 204
|
|
322
329
|
|
|
323
330
|
|
udata/core/user/api_fields.py
CHANGED
|
@@ -61,6 +61,11 @@ user_fields = api.model(
|
|
|
61
61
|
"since": fields.ISODateTime(
|
|
62
62
|
attribute="created_at", description="The registeration date", required=True
|
|
63
63
|
),
|
|
64
|
+
"last_login_at": fields.Raw(
|
|
65
|
+
attribute=lambda o: o.current_login_at if current_user_is_admin_or_self() else None,
|
|
66
|
+
description="The user last connection date (only present for global admins and on /me)",
|
|
67
|
+
readonly=True,
|
|
68
|
+
),
|
|
64
69
|
"page": fields.UrlFor(
|
|
65
70
|
"users.show",
|
|
66
71
|
lambda u: {"user": u},
|
udata/core/user/models.py
CHANGED
|
@@ -82,6 +82,10 @@ class User(WithMetrics, UserMixin, db.Document):
|
|
|
82
82
|
ext = db.MapField(db.GenericEmbeddedDocumentField())
|
|
83
83
|
extras = db.ExtrasField()
|
|
84
84
|
|
|
85
|
+
# Used to track notification for automatic inactive users deletion
|
|
86
|
+
# when YEARS_OF_INACTIVITY_BEFORE_DEACTIVATION is set
|
|
87
|
+
inactive_deletion_notified_at = db.DateTimeField()
|
|
88
|
+
|
|
85
89
|
before_save = Signal()
|
|
86
90
|
after_save = Signal()
|
|
87
91
|
on_create = Signal()
|
|
@@ -237,7 +241,7 @@ class User(WithMetrics, UserMixin, db.Document):
|
|
|
237
241
|
raise NotImplementedError("""This method should not be using directly.
|
|
238
242
|
Use `mark_as_deleted` (or `_delete` if you know what you're doing)""")
|
|
239
243
|
|
|
240
|
-
def mark_as_deleted(self, notify: bool = True):
|
|
244
|
+
def mark_as_deleted(self, notify: bool = True, delete_comments: bool = False):
|
|
241
245
|
if self.avatar.filename is not None:
|
|
242
246
|
storage = storages.avatars
|
|
243
247
|
storage.delete(self.avatar.filename)
|
|
@@ -265,16 +269,17 @@ class User(WithMetrics, UserMixin, db.Document):
|
|
|
265
269
|
member for member in organization.members if member.user != self
|
|
266
270
|
]
|
|
267
271
|
organization.save()
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
discussion.
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
message.
|
|
277
|
-
|
|
272
|
+
if delete_comments:
|
|
273
|
+
for discussion in Discussion.objects(discussion__posted_by=self):
|
|
274
|
+
# Remove all discussions with current user as only participant
|
|
275
|
+
if all(message.posted_by == self for message in discussion.discussion):
|
|
276
|
+
discussion.delete()
|
|
277
|
+
continue
|
|
278
|
+
|
|
279
|
+
for message in discussion.discussion:
|
|
280
|
+
if message.posted_by == self:
|
|
281
|
+
message.content = "DELETED"
|
|
282
|
+
discussion.save()
|
|
278
283
|
Follow.objects(follower=self).delete()
|
|
279
284
|
Follow.objects(following=self).delete()
|
|
280
285
|
|
udata/core/user/tasks.py
CHANGED
|
@@ -1,10 +1,14 @@
|
|
|
1
1
|
import logging
|
|
2
|
+
from copy import copy
|
|
3
|
+
from datetime import datetime, timedelta
|
|
4
|
+
|
|
5
|
+
from flask import current_app
|
|
2
6
|
|
|
3
7
|
from udata import mail
|
|
4
8
|
from udata.i18n import lazy_gettext as _
|
|
5
|
-
from udata.tasks import task
|
|
9
|
+
from udata.tasks import job, task
|
|
6
10
|
|
|
7
|
-
from .models import datastore
|
|
11
|
+
from .models import User, datastore
|
|
8
12
|
|
|
9
13
|
log = logging.getLogger(__name__)
|
|
10
14
|
|
|
@@ -13,3 +17,78 @@ log = logging.getLogger(__name__)
|
|
|
13
17
|
def send_test_mail(email):
|
|
14
18
|
user = datastore.find_user(email=email)
|
|
15
19
|
mail.send(_("Test mail"), user, "test")
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@job("notify-inactive-users")
|
|
23
|
+
def notify_inactive_users(self):
|
|
24
|
+
if not current_app.config["YEARS_OF_INACTIVITY_BEFORE_DEACTIVATION"]:
|
|
25
|
+
logging.warning(
|
|
26
|
+
"YEARS_OF_INACTIVITY_BEFORE_DEACTIVATION setting is not set, no deletion planned"
|
|
27
|
+
)
|
|
28
|
+
return
|
|
29
|
+
notification_comparison_date = (
|
|
30
|
+
datetime.utcnow()
|
|
31
|
+
- timedelta(days=current_app.config["YEARS_OF_INACTIVITY_BEFORE_DEACTIVATION"] * 365)
|
|
32
|
+
+ timedelta(days=current_app.config["DAYS_BEFORE_ACCOUNT_INACTIVITY_NOTIFY_DELAY"])
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
users_to_notify = User.objects(
|
|
36
|
+
deleted=None,
|
|
37
|
+
inactive_deletion_notified_at=None,
|
|
38
|
+
current_login_at__lte=notification_comparison_date,
|
|
39
|
+
)
|
|
40
|
+
for i, user in enumerate(users_to_notify):
|
|
41
|
+
if i >= current_app.config["MAX_NUMBER_OF_USER_INACTIVITY_NOTIFICATIONS"]:
|
|
42
|
+
logging.warning("MAX_NUMBER_OF_USER_INACTIVITY_NOTIFICATIONS reached, stopping here.")
|
|
43
|
+
return
|
|
44
|
+
mail.send(
|
|
45
|
+
_("Inactivity of your {site} account").format(site=current_app.config["SITE_TITLE"]),
|
|
46
|
+
user,
|
|
47
|
+
"account_inactivity",
|
|
48
|
+
user=user,
|
|
49
|
+
)
|
|
50
|
+
logging.debug(f"Notified {user.email} of account inactivity")
|
|
51
|
+
user.inactive_deletion_notified_at = datetime.utcnow()
|
|
52
|
+
user.save()
|
|
53
|
+
|
|
54
|
+
logging.info(f"Notified {users_to_notify.count()} inactive users")
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
@job("delete-inactive-users")
|
|
58
|
+
def delete_inactive_users(self):
|
|
59
|
+
if not current_app.config["YEARS_OF_INACTIVITY_BEFORE_DEACTIVATION"]:
|
|
60
|
+
logging.warning(
|
|
61
|
+
"YEARS_OF_INACTIVITY_BEFORE_DEACTIVATION setting is not set, no deletion planned"
|
|
62
|
+
)
|
|
63
|
+
return
|
|
64
|
+
|
|
65
|
+
# Clear inactive_deletion_notified_at field if user has logged in since notification
|
|
66
|
+
for user in User.objects(deleted=None, inactive_deletion_notified_at__exists=True):
|
|
67
|
+
if user.current_login_at > user.inactive_deletion_notified_at:
|
|
68
|
+
user.inactive_deletion_notified_at = None
|
|
69
|
+
user.save()
|
|
70
|
+
|
|
71
|
+
# Delete inactive users upon notification delay if user still hasn't logged in
|
|
72
|
+
deletion_comparison_date = datetime.utcnow() - timedelta(
|
|
73
|
+
days=current_app.config["YEARS_OF_INACTIVITY_BEFORE_DEACTIVATION"] * 365
|
|
74
|
+
)
|
|
75
|
+
notified_at = datetime.utcnow() - timedelta(
|
|
76
|
+
days=current_app.config["DAYS_BEFORE_ACCOUNT_INACTIVITY_NOTIFY_DELAY"]
|
|
77
|
+
)
|
|
78
|
+
users_to_delete = User.objects(
|
|
79
|
+
deleted=None,
|
|
80
|
+
current_login_at__lte=deletion_comparison_date,
|
|
81
|
+
inactive_deletion_notified_at__lte=notified_at,
|
|
82
|
+
)
|
|
83
|
+
for user in users_to_delete:
|
|
84
|
+
copied_user = copy(user)
|
|
85
|
+
user.mark_as_deleted(notify=False, delete_comments=False)
|
|
86
|
+
logging.warning(f"Deleted user {copied_user.email} due to account inactivity")
|
|
87
|
+
mail.send(
|
|
88
|
+
_("Deletion of your inactive {site} account").format(
|
|
89
|
+
site=current_app.config["SITE_TITLE"]
|
|
90
|
+
),
|
|
91
|
+
copied_user,
|
|
92
|
+
"inactive_account_deleted",
|
|
93
|
+
)
|
|
94
|
+
logging.info(f"Deleted {users_to_delete.count()} inactive users")
|
|
@@ -18,7 +18,7 @@ class UserModelTest:
|
|
|
18
18
|
user = UserFactory()
|
|
19
19
|
other_user = UserFactory()
|
|
20
20
|
org = OrganizationFactory(editors=[user])
|
|
21
|
-
|
|
21
|
+
discussion = DiscussionFactory(
|
|
22
22
|
user=user,
|
|
23
23
|
subject=org,
|
|
24
24
|
discussion=[
|
|
@@ -26,14 +26,6 @@ class UserModelTest:
|
|
|
26
26
|
MessageDiscussionFactory(posted_by=user),
|
|
27
27
|
],
|
|
28
28
|
)
|
|
29
|
-
discussion_with_other = DiscussionFactory(
|
|
30
|
-
user=other_user,
|
|
31
|
-
subject=org,
|
|
32
|
-
discussion=[
|
|
33
|
-
MessageDiscussionFactory(posted_by=other_user),
|
|
34
|
-
MessageDiscussionFactory(posted_by=user),
|
|
35
|
-
],
|
|
36
|
-
)
|
|
37
29
|
user_follow_org = Follow.objects.create(follower=user, following=org)
|
|
38
30
|
user_followed = Follow.objects.create(follower=other_user, following=user)
|
|
39
31
|
|
|
@@ -42,15 +34,40 @@ class UserModelTest:
|
|
|
42
34
|
org.reload()
|
|
43
35
|
assert len(org.members) == 0
|
|
44
36
|
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
assert
|
|
37
|
+
# discussions are kept by default
|
|
38
|
+
discussion.reload()
|
|
39
|
+
assert len(discussion.discussion) == 2
|
|
40
|
+
assert discussion.discussion[1].content != "DELETED"
|
|
48
41
|
|
|
49
42
|
assert Follow.objects(id=user_follow_org.id).first() is None
|
|
50
43
|
assert Follow.objects(id=user_followed.id).first() is None
|
|
51
44
|
|
|
52
45
|
assert user.slug == "deleted"
|
|
53
46
|
|
|
47
|
+
def test_mark_as_deleted_with_comments_deletion(self):
|
|
48
|
+
user = UserFactory()
|
|
49
|
+
other_user = UserFactory()
|
|
50
|
+
discussion_only_user = DiscussionFactory(
|
|
51
|
+
user=user,
|
|
52
|
+
discussion=[
|
|
53
|
+
MessageDiscussionFactory(posted_by=user),
|
|
54
|
+
MessageDiscussionFactory(posted_by=user),
|
|
55
|
+
],
|
|
56
|
+
)
|
|
57
|
+
discussion_with_other = DiscussionFactory(
|
|
58
|
+
user=other_user,
|
|
59
|
+
discussion=[
|
|
60
|
+
MessageDiscussionFactory(posted_by=other_user),
|
|
61
|
+
MessageDiscussionFactory(posted_by=user),
|
|
62
|
+
],
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
user.mark_as_deleted(delete_comments=True)
|
|
66
|
+
|
|
67
|
+
assert Discussion.objects(id=discussion_only_user.id).first() is None
|
|
68
|
+
discussion_with_other.reload()
|
|
69
|
+
assert discussion_with_other.discussion[1].content == "DELETED"
|
|
70
|
+
|
|
54
71
|
def test_mark_as_deleted_slug_multiple(self):
|
|
55
72
|
user = UserFactory()
|
|
56
73
|
other_user = UserFactory()
|
udata/features/transfer/api.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
from flask import request
|
|
1
|
+
from flask import abort, request
|
|
2
2
|
|
|
3
3
|
from udata.api import API, api, base_reference, fields
|
|
4
4
|
from udata.core.dataservices.models import Dataservice
|
|
@@ -27,7 +27,7 @@ transfer_request_fields = api.model(
|
|
|
27
27
|
"recipient": fields.Nested(
|
|
28
28
|
base_reference,
|
|
29
29
|
required=True,
|
|
30
|
-
description=("The transfer recipient,
|
|
30
|
+
description=("The transfer recipient, either an user or an organization"),
|
|
31
31
|
),
|
|
32
32
|
"comment": fields.String(
|
|
33
33
|
description="An explanation about the transfer request", required=True
|
|
@@ -68,12 +68,12 @@ transfer_fields = api.model(
|
|
|
68
68
|
"owner": fields.Polymorph(
|
|
69
69
|
person_mapping,
|
|
70
70
|
readonly=True,
|
|
71
|
-
description=("The user or organization currently owning
|
|
71
|
+
description=("The user or organization currently owning the transfered object"),
|
|
72
72
|
),
|
|
73
73
|
"recipient": fields.Polymorph(
|
|
74
74
|
person_mapping,
|
|
75
75
|
readonly=True,
|
|
76
|
-
description=("The user or organization receiving
|
|
76
|
+
description=("The user or organization receiving the transfered object"),
|
|
77
77
|
),
|
|
78
78
|
"subject": fields.Polymorph(
|
|
79
79
|
subject_mapping, readonly=True, description="The transfered object"
|
|
@@ -186,6 +186,9 @@ class TransferRequestAPI(API):
|
|
|
186
186
|
"""Respond to a transfer request"""
|
|
187
187
|
transfer = Transfer.objects.get_or_404(id=id_or_404(id))
|
|
188
188
|
|
|
189
|
+
if transfer.status != "pending":
|
|
190
|
+
abort(400, "Cannot update transfer after accepting/refusing")
|
|
191
|
+
|
|
189
192
|
data = request.json
|
|
190
193
|
comment = data.get("comment")
|
|
191
194
|
|