udata 10.4.1.dev35201__py2.py3-none-any.whl → 10.4.2__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of udata might be problematic. Click here for more details.
- udata/__init__.py +1 -1
- udata/core/activity/__init__.py +2 -0
- udata/core/activity/api.py +10 -2
- udata/core/activity/models.py +28 -1
- udata/core/activity/tasks.py +19 -4
- udata/core/dataservices/activities.py +53 -0
- udata/core/dataservices/api.py +43 -0
- udata/core/dataservices/models.py +16 -20
- udata/core/dataset/activities.py +52 -5
- udata/core/dataset/api.py +44 -0
- udata/core/dataset/csv.py +0 -1
- udata/core/dataset/models.py +49 -47
- udata/core/dataset/rdf.py +1 -1
- udata/core/metrics/commands.py +1 -0
- udata/core/metrics/helpers.py +102 -0
- udata/core/metrics/models.py +1 -0
- udata/core/metrics/tasks.py +1 -0
- udata/core/organization/activities.py +3 -2
- udata/core/organization/api.py +11 -0
- udata/core/organization/api_fields.py +6 -5
- udata/core/organization/models.py +31 -31
- udata/core/owned.py +1 -1
- udata/core/post/api.py +34 -0
- udata/core/reuse/activities.py +6 -5
- udata/core/reuse/api.py +42 -1
- udata/core/reuse/models.py +8 -16
- udata/core/site/models.py +33 -0
- udata/core/topic/activities.py +36 -0
- udata/core/topic/models.py +23 -15
- udata/core/user/activities.py +17 -6
- udata/core/user/api.py +1 -0
- udata/core/user/api_fields.py +6 -1
- udata/core/user/models.py +39 -32
- udata/migrations/2025-05-22-purge-duplicate-activities.py +101 -0
- udata/mongo/datetime_fields.py +1 -0
- udata/settings.py +4 -0
- udata/tests/api/test_activities_api.py +29 -1
- udata/tests/api/test_dataservices_api.py +53 -0
- udata/tests/api/test_datasets_api.py +61 -0
- udata/tests/api/test_organizations_api.py +27 -2
- udata/tests/api/test_reuses_api.py +54 -0
- udata/tests/dataset/test_csv_adapter.py +6 -3
- udata/tests/dataset/test_dataset_model.py +49 -0
- udata/tests/test_topics.py +19 -0
- {udata-10.4.1.dev35201.dist-info → udata-10.4.2.dist-info}/METADATA +17 -2
- {udata-10.4.1.dev35201.dist-info → udata-10.4.2.dist-info}/RECORD +50 -46
- {udata-10.4.1.dev35201.dist-info → udata-10.4.2.dist-info}/LICENSE +0 -0
- {udata-10.4.1.dev35201.dist-info → udata-10.4.2.dist-info}/WHEEL +0 -0
- {udata-10.4.1.dev35201.dist-info → udata-10.4.2.dist-info}/entry_points.txt +0 -0
- {udata-10.4.1.dev35201.dist-info → udata-10.4.2.dist-info}/top_level.txt +0 -0
udata/core/reuse/models.py
CHANGED
|
@@ -3,6 +3,7 @@ from mongoengine.signals import post_save, pre_save
|
|
|
3
3
|
from werkzeug.utils import cached_property
|
|
4
4
|
|
|
5
5
|
from udata.api_fields import field, function_field, generate_fields
|
|
6
|
+
from udata.core.activity.models import Auditable
|
|
6
7
|
from udata.core.dataset.api_fields import dataset_fields
|
|
7
8
|
from udata.core.owned import Owned, OwnedQuerySet
|
|
8
9
|
from udata.core.reuse.api_fields import BIGGEST_IMAGE_SIZE
|
|
@@ -60,7 +61,7 @@ class ReuseBadgeMixin(BadgeMixin):
|
|
|
60
61
|
additional_filters={"organization_badge": "organization.badges"},
|
|
61
62
|
mask="*,datasets{id,title,uri,page}",
|
|
62
63
|
)
|
|
63
|
-
class Reuse(db.Datetimed, WithMetrics, ReuseBadgeMixin, Owned, db.Document):
|
|
64
|
+
class Reuse(db.Datetimed, Auditable, WithMetrics, ReuseBadgeMixin, Owned, db.Document):
|
|
64
65
|
title = field(
|
|
65
66
|
db.StringField(required=True),
|
|
66
67
|
sortable=True,
|
|
@@ -71,6 +72,7 @@ class Reuse(db.Datetimed, WithMetrics, ReuseBadgeMixin, Owned, db.Document):
|
|
|
71
72
|
max_length=255, required=True, populate_from="title", update=True, follow=True
|
|
72
73
|
),
|
|
73
74
|
readonly=True,
|
|
75
|
+
auditable=False,
|
|
74
76
|
)
|
|
75
77
|
description = field(
|
|
76
78
|
db.StringField(required=True),
|
|
@@ -126,15 +128,17 @@ class Reuse(db.Datetimed, WithMetrics, ReuseBadgeMixin, Owned, db.Document):
|
|
|
126
128
|
private = field(db.BooleanField(default=False), filterable={})
|
|
127
129
|
|
|
128
130
|
ext = db.MapField(db.GenericEmbeddedDocumentField())
|
|
129
|
-
extras = field(db.ExtrasField())
|
|
131
|
+
extras = field(db.ExtrasField(), auditable=False)
|
|
130
132
|
|
|
131
133
|
featured = field(
|
|
132
134
|
db.BooleanField(),
|
|
133
135
|
filterable={},
|
|
134
136
|
readonly=True,
|
|
137
|
+
auditable=False,
|
|
135
138
|
)
|
|
136
139
|
deleted = field(
|
|
137
140
|
db.DateTimeField(),
|
|
141
|
+
auditable=False,
|
|
138
142
|
)
|
|
139
143
|
archived = field(
|
|
140
144
|
db.DateTimeField(),
|
|
@@ -181,18 +185,6 @@ class Reuse(db.Datetimed, WithMetrics, ReuseBadgeMixin, Owned, db.Document):
|
|
|
181
185
|
# Emit before_save
|
|
182
186
|
cls.before_save.send(document)
|
|
183
187
|
|
|
184
|
-
@classmethod
|
|
185
|
-
def post_save(cls, sender, document, **kwargs):
|
|
186
|
-
if "post_save" in kwargs.get("ignores", []):
|
|
187
|
-
return
|
|
188
|
-
cls.after_save.send(document)
|
|
189
|
-
if kwargs.get("created"):
|
|
190
|
-
cls.on_create.send(document)
|
|
191
|
-
else:
|
|
192
|
-
cls.on_update.send(document)
|
|
193
|
-
if document.deleted:
|
|
194
|
-
cls.on_delete.send(document)
|
|
195
|
-
|
|
196
188
|
def url_for(self, *args, **kwargs):
|
|
197
189
|
return endpoint_for("reuses.show", "api.reuse", reuse=self, *args, **kwargs)
|
|
198
190
|
|
|
@@ -289,13 +281,13 @@ class Reuse(db.Datetimed, WithMetrics, ReuseBadgeMixin, Owned, db.Document):
|
|
|
289
281
|
from udata.models import Discussion
|
|
290
282
|
|
|
291
283
|
self.metrics["discussions"] = Discussion.objects(subject=self, closed=None).count()
|
|
292
|
-
self.save()
|
|
284
|
+
self.save(signal_kwargs={"ignores": ["post_save"]})
|
|
293
285
|
|
|
294
286
|
def count_followers(self):
|
|
295
287
|
from udata.models import Follow
|
|
296
288
|
|
|
297
289
|
self.metrics["followers"] = Follow.objects(until=None).followers(self).count()
|
|
298
|
-
self.save()
|
|
290
|
+
self.save(signal_kwargs={"ignores": ["post_save"]})
|
|
299
291
|
|
|
300
292
|
|
|
301
293
|
pre_save.connect(Reuse.pre_save, sender=Reuse)
|
udata/core/site/models.py
CHANGED
|
@@ -3,6 +3,7 @@ from werkzeug.local import LocalProxy
|
|
|
3
3
|
|
|
4
4
|
from udata.core.dataservices.models import Dataservice
|
|
5
5
|
from udata.core.dataset.models import Dataset
|
|
6
|
+
from udata.core.metrics.helpers import get_metrics_for_model, get_stock_metrics
|
|
6
7
|
from udata.core.organization.models import Organization
|
|
7
8
|
from udata.core.reuse.models import Reuse
|
|
8
9
|
from udata.models import WithMetrics, db
|
|
@@ -36,15 +37,23 @@ class Site(WithMetrics, db.Document):
|
|
|
36
37
|
"max_org_reuses",
|
|
37
38
|
"max_org_datasets",
|
|
38
39
|
"datasets",
|
|
40
|
+
"datasets_visits_by_months",
|
|
39
41
|
"discussions",
|
|
40
42
|
"followers",
|
|
41
43
|
"organizations",
|
|
42
44
|
"public-service",
|
|
43
45
|
"resources",
|
|
46
|
+
"resources_downloads_by_months",
|
|
44
47
|
"reuses",
|
|
45
48
|
"dataservices",
|
|
46
49
|
"users",
|
|
47
50
|
"harvesters",
|
|
51
|
+
"users_by_months",
|
|
52
|
+
"datasets_by_months",
|
|
53
|
+
"harvesters_by_months",
|
|
54
|
+
"reuses_by_months",
|
|
55
|
+
"organizations_by_months",
|
|
56
|
+
"discussions_by_months",
|
|
48
57
|
]
|
|
49
58
|
|
|
50
59
|
def __str__(self):
|
|
@@ -72,6 +81,9 @@ class Site(WithMetrics, db.Document):
|
|
|
72
81
|
from udata.models import Dataset
|
|
73
82
|
|
|
74
83
|
self.metrics["datasets"] = Dataset.objects.visible().count()
|
|
84
|
+
self.metrics["datasets_visits_by_months"] = get_metrics_for_model(
|
|
85
|
+
"site", None, ["visit_dataset"]
|
|
86
|
+
)[0]
|
|
75
87
|
self.save()
|
|
76
88
|
|
|
77
89
|
def count_resources(self):
|
|
@@ -83,6 +95,9 @@ class Site(WithMetrics, db.Document):
|
|
|
83
95
|
),
|
|
84
96
|
{},
|
|
85
97
|
).get("count", 0)
|
|
98
|
+
self.metrics["resources_downloads_by_months"] = get_metrics_for_model(
|
|
99
|
+
"site", None, ["download_resource"]
|
|
100
|
+
)[0]
|
|
86
101
|
self.save()
|
|
87
102
|
|
|
88
103
|
def count_reuses(self):
|
|
@@ -172,6 +187,24 @@ class Site(WithMetrics, db.Document):
|
|
|
172
187
|
self.metrics["max_org_datasets"] = org.metrics["datasets"] if org else 0
|
|
173
188
|
self.save()
|
|
174
189
|
|
|
190
|
+
def count_stock_metrics(self):
|
|
191
|
+
from udata.harvest.models import HarvestSource
|
|
192
|
+
from udata.models import Discussion, User
|
|
193
|
+
|
|
194
|
+
self.metrics["users_by_months"] = get_stock_metrics(User.objects())
|
|
195
|
+
self.metrics["datasets_by_months"] = get_stock_metrics(
|
|
196
|
+
Dataset.objects().visible(), date_label="created_at_internal"
|
|
197
|
+
)
|
|
198
|
+
self.metrics["harvesters_by_months"] = get_stock_metrics(HarvestSource.objects())
|
|
199
|
+
self.metrics["reuses_by_months"] = get_stock_metrics(Reuse.objects().visible())
|
|
200
|
+
self.metrics["organizations_by_months"] = get_stock_metrics(
|
|
201
|
+
Organization.objects().visible()
|
|
202
|
+
)
|
|
203
|
+
self.metrics["discussions_by_months"] = get_stock_metrics(
|
|
204
|
+
Discussion.objects(), date_label="created"
|
|
205
|
+
)
|
|
206
|
+
self.save()
|
|
207
|
+
|
|
175
208
|
|
|
176
209
|
def get_current_site():
|
|
177
210
|
if getattr(g, "site", None) is None:
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
from flask_security import current_user
|
|
2
|
+
|
|
3
|
+
from udata.i18n import lazy_gettext as _
|
|
4
|
+
from udata.models import Activity, Topic, db
|
|
5
|
+
|
|
6
|
+
__all__ = ("UserCreatedTopic", "UserUpdatedTopic", "TopicRelatedActivity")
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class TopicRelatedActivity(object):
|
|
10
|
+
related_to = db.ReferenceField("Topic")
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class UserCreatedTopic(TopicRelatedActivity, Activity):
|
|
14
|
+
key = "topic:created"
|
|
15
|
+
icon = "fa fa-plus"
|
|
16
|
+
badge_type = "success"
|
|
17
|
+
label = _("created a topic")
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class UserUpdatedTopic(TopicRelatedActivity, Activity):
|
|
21
|
+
key = "topic:updated"
|
|
22
|
+
icon = "fa fa-pencil"
|
|
23
|
+
label = _("updated a topic")
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@Topic.on_create.connect
|
|
27
|
+
def on_user_created_topic(topic):
|
|
28
|
+
if current_user and current_user.is_authenticated:
|
|
29
|
+
UserCreatedTopic.emit(topic, topic.organization)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@Topic.on_update.connect
|
|
33
|
+
def on_user_updated_topic(topic, **kwargs):
|
|
34
|
+
changed_fields = kwargs.get("changed_fields", [])
|
|
35
|
+
if current_user and current_user.is_authenticated:
|
|
36
|
+
UserUpdatedTopic.emit(topic, topic.organization, changed_fields)
|
udata/core/topic/models.py
CHANGED
|
@@ -1,6 +1,9 @@
|
|
|
1
|
+
from blinker import Signal
|
|
1
2
|
from flask import url_for
|
|
2
|
-
from mongoengine.signals import pre_save
|
|
3
|
+
from mongoengine.signals import post_save, pre_save
|
|
3
4
|
|
|
5
|
+
from udata.api_fields import field
|
|
6
|
+
from udata.core.activity.models import Auditable
|
|
4
7
|
from udata.core.owned import Owned, OwnedQuerySet
|
|
5
8
|
from udata.models import SpatialCoverage, db
|
|
6
9
|
from udata.search import reindex
|
|
@@ -8,24 +11,24 @@ from udata.search import reindex
|
|
|
8
11
|
__all__ = ("Topic",)
|
|
9
12
|
|
|
10
13
|
|
|
11
|
-
class Topic(db.
|
|
12
|
-
name = db.StringField(required=True)
|
|
13
|
-
slug =
|
|
14
|
-
max_length=255, required=True, populate_from="name", update=True, follow=True
|
|
14
|
+
class Topic(db.Datetimed, Auditable, db.Document, Owned):
|
|
15
|
+
name = field(db.StringField(required=True))
|
|
16
|
+
slug = field(
|
|
17
|
+
db.SlugField(max_length=255, required=True, populate_from="name", update=True, follow=True),
|
|
18
|
+
auditable=False,
|
|
15
19
|
)
|
|
16
|
-
description = db.StringField()
|
|
17
|
-
tags = db.ListField(db.StringField())
|
|
18
|
-
color = db.IntField()
|
|
20
|
+
description = field(db.StringField())
|
|
21
|
+
tags = field(db.ListField(db.StringField()))
|
|
22
|
+
color = field(db.IntField())
|
|
19
23
|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
reuses = db.ListField(db.LazyReferenceField("Reuse", reverse_delete_rule=db.PULL))
|
|
24
|
+
datasets = field(db.ListField(db.LazyReferenceField("Dataset", reverse_delete_rule=db.PULL)))
|
|
25
|
+
reuses = field(db.ListField(db.LazyReferenceField("Reuse", reverse_delete_rule=db.PULL)))
|
|
23
26
|
|
|
24
|
-
featured = db.BooleanField(default=False)
|
|
25
|
-
private = db.BooleanField()
|
|
26
|
-
extras = db.ExtrasField()
|
|
27
|
+
featured = field(db.BooleanField(default=False), auditable=False)
|
|
28
|
+
private = field(db.BooleanField())
|
|
29
|
+
extras = field(db.ExtrasField(), auditable=False)
|
|
27
30
|
|
|
28
|
-
spatial = db.EmbeddedDocumentField(SpatialCoverage)
|
|
31
|
+
spatial = field(db.EmbeddedDocumentField(SpatialCoverage))
|
|
29
32
|
|
|
30
33
|
meta = {
|
|
31
34
|
"indexes": ["$name", "created_at", "slug"] + Owned.meta["indexes"],
|
|
@@ -34,6 +37,10 @@ class Topic(db.Document, Owned, db.Datetimed):
|
|
|
34
37
|
"queryset_class": OwnedQuerySet,
|
|
35
38
|
}
|
|
36
39
|
|
|
40
|
+
after_save = Signal()
|
|
41
|
+
on_create = Signal()
|
|
42
|
+
on_update = Signal()
|
|
43
|
+
|
|
37
44
|
def __str__(self):
|
|
38
45
|
return self.name
|
|
39
46
|
|
|
@@ -60,3 +67,4 @@ class Topic(db.Document, Owned, db.Datetimed):
|
|
|
60
67
|
|
|
61
68
|
|
|
62
69
|
pre_save.connect(Topic.pre_save, sender=Topic)
|
|
70
|
+
post_save.connect(Topic.post_save, sender=Topic)
|
udata/core/user/activities.py
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
from flask_security import current_user
|
|
2
2
|
|
|
3
|
+
from udata.core.dataservices.activities import DataserviceRelatedActivity
|
|
4
|
+
from udata.core.dataservices.models import Dataservice
|
|
3
5
|
from udata.core.dataset.activities import DatasetRelatedActivity
|
|
4
6
|
from udata.core.discussions.signals import on_new_discussion, on_new_discussion_comment
|
|
5
7
|
from udata.core.followers.signals import on_follow
|
|
@@ -28,11 +30,6 @@ class DiscussActivity(object):
|
|
|
28
30
|
badge_type = "warning"
|
|
29
31
|
|
|
30
32
|
|
|
31
|
-
class UserStarredOrganization(FollowActivity, OrgRelatedActivity, Activity):
|
|
32
|
-
key = "organization:followed"
|
|
33
|
-
label = _("followed an organization")
|
|
34
|
-
|
|
35
|
-
|
|
36
33
|
class UserFollowedUser(FollowActivity, Activity):
|
|
37
34
|
key = "user:followed"
|
|
38
35
|
label = _("followed a user")
|
|
@@ -40,6 +37,11 @@ class UserFollowedUser(FollowActivity, Activity):
|
|
|
40
37
|
template = "activity/user.html"
|
|
41
38
|
|
|
42
39
|
|
|
40
|
+
class UserDiscussedDataservice(DiscussActivity, DataserviceRelatedActivity, Activity):
|
|
41
|
+
key = "dataservice:discussed"
|
|
42
|
+
label = _("discussed a dataservice")
|
|
43
|
+
|
|
44
|
+
|
|
43
45
|
class UserDiscussedDataset(DiscussActivity, DatasetRelatedActivity, Activity):
|
|
44
46
|
key = "dataset:discussed"
|
|
45
47
|
label = _("discussed a dataset")
|
|
@@ -50,6 +52,11 @@ class UserDiscussedReuse(DiscussActivity, ReuseRelatedActivity, Activity):
|
|
|
50
52
|
label = _("discussed a reuse")
|
|
51
53
|
|
|
52
54
|
|
|
55
|
+
class UserFollowedDataservice(FollowActivity, DataserviceRelatedActivity, Activity):
|
|
56
|
+
key = "dataservice:followed"
|
|
57
|
+
label = _("followed a dataservice")
|
|
58
|
+
|
|
59
|
+
|
|
53
60
|
class UserFollowedDataset(FollowActivity, DatasetRelatedActivity, Activity):
|
|
54
61
|
key = "dataset:followed"
|
|
55
62
|
label = _("followed a dataset")
|
|
@@ -68,7 +75,9 @@ class UserFollowedOrganization(FollowActivity, OrgRelatedActivity, Activity):
|
|
|
68
75
|
@on_follow.connect
|
|
69
76
|
def write_activity_on_follow(follow, **kwargs):
|
|
70
77
|
if current_user.is_authenticated:
|
|
71
|
-
if isinstance(follow.following,
|
|
78
|
+
if isinstance(follow.following, Dataservice):
|
|
79
|
+
UserFollowedDataservice.emit(follow.following)
|
|
80
|
+
elif isinstance(follow.following, Dataset):
|
|
72
81
|
UserFollowedDataset.emit(follow.following)
|
|
73
82
|
elif isinstance(follow.following, Reuse):
|
|
74
83
|
UserFollowedReuse.emit(follow.following)
|
|
@@ -82,6 +91,8 @@ def write_activity_on_follow(follow, **kwargs):
|
|
|
82
91
|
@on_new_discussion_comment.connect
|
|
83
92
|
def write_activity_on_discuss(discussion, **kwargs):
|
|
84
93
|
if current_user.is_authenticated:
|
|
94
|
+
if isinstance(discussion.subject, Dataservice):
|
|
95
|
+
UserDiscussedDataservice.emit(discussion.subject)
|
|
85
96
|
if isinstance(discussion.subject, Dataset):
|
|
86
97
|
UserDiscussedDataset.emit(discussion.subject)
|
|
87
98
|
elif isinstance(discussion.subject, Reuse):
|
udata/core/user/api.py
CHANGED
udata/core/user/api_fields.py
CHANGED
|
@@ -30,7 +30,7 @@ user_ref_fields = api.inherit(
|
|
|
30
30
|
},
|
|
31
31
|
)
|
|
32
32
|
|
|
33
|
-
from udata.core.organization.api_fields import org_ref_fields # noqa
|
|
33
|
+
from udata.core.organization.api_fields import member_email_with_visibility_check, org_ref_fields # noqa
|
|
34
34
|
|
|
35
35
|
user_fields = api.model(
|
|
36
36
|
"User",
|
|
@@ -126,6 +126,11 @@ user_suggestion_fields = api.model(
|
|
|
126
126
|
"avatar_url": fields.ImageField(
|
|
127
127
|
size=BIGGEST_AVATAR_SIZE, description="The user avatar URL", readonly=True
|
|
128
128
|
),
|
|
129
|
+
"email": fields.Raw(
|
|
130
|
+
attribute=lambda o: member_email_with_visibility_check(o["email"]),
|
|
131
|
+
description="The user email (only the domain for non-admin user)",
|
|
132
|
+
readonly=True,
|
|
133
|
+
),
|
|
129
134
|
"slug": fields.String(description="The user permalink string", readonly=True),
|
|
130
135
|
},
|
|
131
136
|
)
|
udata/core/user/models.py
CHANGED
|
@@ -12,6 +12,7 @@ from mongoengine.signals import post_save, pre_save
|
|
|
12
12
|
from werkzeug.utils import cached_property
|
|
13
13
|
|
|
14
14
|
from udata import mail
|
|
15
|
+
from udata.api_fields import field
|
|
15
16
|
from udata.core import storages
|
|
16
17
|
from udata.core.discussions.models import Discussion
|
|
17
18
|
from udata.core.storages import avatars, default_image_basename
|
|
@@ -42,49 +43,53 @@ class UserSettings(db.EmbeddedDocument):
|
|
|
42
43
|
|
|
43
44
|
|
|
44
45
|
class User(WithMetrics, UserMixin, db.Document):
|
|
45
|
-
slug =
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
46
|
+
slug = field(
|
|
47
|
+
db.SlugField(max_length=255, required=True, populate_from="fullname"), auditable=False
|
|
48
|
+
)
|
|
49
|
+
email = field(db.StringField(max_length=255, required=True, unique=True))
|
|
50
|
+
password = field(db.StringField())
|
|
51
|
+
active = field(db.BooleanField())
|
|
52
|
+
fs_uniquifier = field(db.StringField(max_length=64, unique=True, sparse=True))
|
|
53
|
+
roles = field(db.ListField(db.ReferenceField(Role), default=[]))
|
|
51
54
|
|
|
52
|
-
first_name = db.StringField(max_length=255, required=True)
|
|
53
|
-
last_name = db.StringField(max_length=255, required=True)
|
|
55
|
+
first_name = field(db.StringField(max_length=255, required=True))
|
|
56
|
+
last_name = field(db.StringField(max_length=255, required=True))
|
|
54
57
|
|
|
55
|
-
avatar_url = db.URLField()
|
|
56
|
-
avatar =
|
|
57
|
-
|
|
58
|
-
|
|
58
|
+
avatar_url = field(db.URLField())
|
|
59
|
+
avatar = field(
|
|
60
|
+
db.ImageField(fs=avatars, basename=default_image_basename, thumbnails=AVATAR_SIZES)
|
|
61
|
+
)
|
|
62
|
+
website = field(db.URLField())
|
|
63
|
+
about = field(db.StringField())
|
|
59
64
|
|
|
60
|
-
prefered_language = db.StringField()
|
|
65
|
+
prefered_language = field(db.StringField())
|
|
61
66
|
|
|
62
|
-
apikey = db.StringField()
|
|
67
|
+
apikey = field(db.StringField())
|
|
63
68
|
|
|
64
|
-
created_at = db.DateTimeField(default=datetime.utcnow, required=True)
|
|
69
|
+
created_at = field(db.DateTimeField(default=datetime.utcnow, required=True), auditable=False)
|
|
65
70
|
|
|
66
71
|
# The field below is required for Flask-security
|
|
67
72
|
# when SECURITY_CONFIRMABLE is True
|
|
68
|
-
confirmed_at = db.DateTimeField()
|
|
73
|
+
confirmed_at = field(db.DateTimeField(), auditable=False)
|
|
69
74
|
|
|
70
|
-
password_rotation_demanded = db.DateTimeField()
|
|
71
|
-
password_rotation_performed = db.DateTimeField()
|
|
75
|
+
password_rotation_demanded = field(db.DateTimeField(), auditable=False)
|
|
76
|
+
password_rotation_performed = field(db.DateTimeField(), auditable=False)
|
|
72
77
|
|
|
73
78
|
# The 5 fields below are required for Flask-security
|
|
74
79
|
# when SECURITY_TRACKABLE is True
|
|
75
|
-
last_login_at = db.DateTimeField()
|
|
76
|
-
current_login_at = db.DateTimeField()
|
|
77
|
-
last_login_ip = db.StringField()
|
|
78
|
-
current_login_ip = db.StringField()
|
|
79
|
-
login_count = db.IntField()
|
|
80
|
+
last_login_at = field(db.DateTimeField(), auditable=False)
|
|
81
|
+
current_login_at = field(db.DateTimeField(), auditable=False)
|
|
82
|
+
last_login_ip = field(db.StringField(), auditable=False)
|
|
83
|
+
current_login_ip = field(db.StringField(), auditable=False)
|
|
84
|
+
login_count = field(db.IntField(), auditable=False)
|
|
80
85
|
|
|
81
|
-
deleted = db.DateTimeField()
|
|
82
|
-
ext = db.MapField(db.GenericEmbeddedDocumentField())
|
|
83
|
-
extras = db.ExtrasField()
|
|
86
|
+
deleted = field(db.DateTimeField())
|
|
87
|
+
ext = field(db.MapField(db.GenericEmbeddedDocumentField()))
|
|
88
|
+
extras = field(db.ExtrasField(), auditable=False)
|
|
84
89
|
|
|
85
90
|
# Used to track notification for automatic inactive users deletion
|
|
86
91
|
# when YEARS_OF_INACTIVITY_BEFORE_DELETION is set
|
|
87
|
-
inactive_deletion_notified_at = db.DateTimeField()
|
|
92
|
+
inactive_deletion_notified_at = field(db.DateTimeField(), auditable=False)
|
|
88
93
|
|
|
89
94
|
before_save = Signal()
|
|
90
95
|
after_save = Signal()
|
|
@@ -209,6 +214,8 @@ class User(WithMetrics, UserMixin, db.Document):
|
|
|
209
214
|
|
|
210
215
|
@classmethod
|
|
211
216
|
def post_save(cls, sender, document, **kwargs):
|
|
217
|
+
if "post_save" in kwargs.get("ignores", []):
|
|
218
|
+
return
|
|
212
219
|
cls.after_save.send(document)
|
|
213
220
|
if kwargs.get("created"):
|
|
214
221
|
cls.on_create.send(document)
|
|
@@ -294,31 +301,31 @@ class User(WithMetrics, UserMixin, db.Document):
|
|
|
294
301
|
from udata.models import Dataset
|
|
295
302
|
|
|
296
303
|
self.metrics["datasets"] = Dataset.objects(owner=self).visible().count()
|
|
297
|
-
self.save()
|
|
304
|
+
self.save(signal_kwargs={"ignores": ["post_save"]})
|
|
298
305
|
|
|
299
306
|
def count_reuses(self):
|
|
300
307
|
from udata.models import Reuse
|
|
301
308
|
|
|
302
309
|
self.metrics["reuses"] = Reuse.objects(owner=self).visible().count()
|
|
303
|
-
self.save()
|
|
310
|
+
self.save(signal_kwargs={"ignores": ["post_save"]})
|
|
304
311
|
|
|
305
312
|
def count_dataservices(self):
|
|
306
313
|
from udata.core.dataservices.models import Dataservice
|
|
307
314
|
|
|
308
315
|
self.metrics["dataservices"] = Dataservice.objects(owner=self).visible().count()
|
|
309
|
-
self.save()
|
|
316
|
+
self.save(signal_kwargs={"ignores": ["post_save"]})
|
|
310
317
|
|
|
311
318
|
def count_followers(self):
|
|
312
319
|
from udata.models import Follow
|
|
313
320
|
|
|
314
321
|
self.metrics["followers"] = Follow.objects(until=None).followers(self).count()
|
|
315
|
-
self.save()
|
|
322
|
+
self.save(signal_kwargs={"ignores": ["post_save"]})
|
|
316
323
|
|
|
317
324
|
def count_following(self):
|
|
318
325
|
from udata.models import Follow
|
|
319
326
|
|
|
320
327
|
self.metrics["following"] = Follow.objects.following(self).count()
|
|
321
|
-
self.save()
|
|
328
|
+
self.save(signal_kwargs={"ignores": ["post_save"]})
|
|
322
329
|
|
|
323
330
|
|
|
324
331
|
datastore = MongoEngineUserDatastore(db, User, Role)
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
"""
|
|
2
|
+
This migration updates Topic.featured to False when it is None.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
import logging
|
|
6
|
+
from datetime import datetime, timedelta
|
|
7
|
+
|
|
8
|
+
from mongoengine.connection import get_db
|
|
9
|
+
|
|
10
|
+
from udata.core.dataset.activities import UserCreatedDataset, UserDeletedDataset, UserUpdatedDataset
|
|
11
|
+
from udata.core.organization.activities import UserUpdatedOrganization
|
|
12
|
+
from udata.core.reuse.activities import UserCreatedReuse, UserDeletedReuse, UserUpdatedReuse
|
|
13
|
+
from udata.core.user.activities import (
|
|
14
|
+
UserDiscussedDataset,
|
|
15
|
+
UserDiscussedReuse,
|
|
16
|
+
UserFollowedDataset,
|
|
17
|
+
UserFollowedOrganization,
|
|
18
|
+
UserFollowedReuse,
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
log = logging.getLogger(__name__)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def migrate(db):
|
|
25
|
+
# Remove legacy fields (`as_organization`, `kwargs`) from old activities
|
|
26
|
+
result = get_db().activity.update_many({}, {"$unset": {"as_organization": ""}})
|
|
27
|
+
log.info(
|
|
28
|
+
f"Legacy field `as_organization` removed from {result.modified_count} activity objects"
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
result = get_db().activity.update_many({}, {"$unset": {"kwargs": ""}})
|
|
32
|
+
log.info(f"Legacy field `kwargs` removed from {result.modified_count} activity objects")
|
|
33
|
+
|
|
34
|
+
# Clean duplicate activities in case of discussion or following
|
|
35
|
+
# - remove the "updated" activity on the discussed/followed object
|
|
36
|
+
# - remove the activity on the organization
|
|
37
|
+
# The heuristic is to look for specific activities by the same actor on the targeted object
|
|
38
|
+
# within a -1 +1 second timespan
|
|
39
|
+
for action_related_activity, object_updated_activity in [
|
|
40
|
+
(UserDiscussedDataset, UserUpdatedDataset),
|
|
41
|
+
(UserDiscussedReuse, UserUpdatedReuse),
|
|
42
|
+
(UserFollowedDataset, UserUpdatedDataset),
|
|
43
|
+
(UserFollowedReuse, UserUpdatedReuse),
|
|
44
|
+
]:
|
|
45
|
+
org_activity_count = 0
|
|
46
|
+
object_activity_count = 0
|
|
47
|
+
activities = (
|
|
48
|
+
action_related_activity.objects()
|
|
49
|
+
.no_dereference() # We use no_dereference in query to prevent DBref DoesNotExist errors
|
|
50
|
+
.no_cache()
|
|
51
|
+
.timeout(False)
|
|
52
|
+
)
|
|
53
|
+
log.info(
|
|
54
|
+
f"{datetime.utcnow()}: Processing {activities.count()} {action_related_activity} activities..."
|
|
55
|
+
)
|
|
56
|
+
for act in activities:
|
|
57
|
+
object_activity_count += object_updated_activity.objects(
|
|
58
|
+
actor=act.actor.id,
|
|
59
|
+
related_to=act.related_to.id,
|
|
60
|
+
created_at__gte=act.created_at - timedelta(seconds=1),
|
|
61
|
+
created_at__lte=act.created_at + timedelta(seconds=1),
|
|
62
|
+
).delete()
|
|
63
|
+
if act.organization:
|
|
64
|
+
org_activity_count += UserUpdatedOrganization.objects(
|
|
65
|
+
actor=act.actor.id,
|
|
66
|
+
related_to=act.organization,
|
|
67
|
+
created_at__gte=act.created_at - timedelta(seconds=1),
|
|
68
|
+
created_at__lte=act.created_at + timedelta(seconds=1),
|
|
69
|
+
).delete()
|
|
70
|
+
log.info(
|
|
71
|
+
f"{datetime.utcnow()}: Deleted {object_activity_count} {object_updated_activity} and {org_activity_count} UserUpdatedOrganization activities"
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
# Clean duplicated UserUpdatedOrganization activities on organization for any object related activity
|
|
75
|
+
for object_related_activity in [
|
|
76
|
+
UserCreatedDataset,
|
|
77
|
+
UserUpdatedDataset,
|
|
78
|
+
UserDeletedDataset,
|
|
79
|
+
UserCreatedReuse,
|
|
80
|
+
UserUpdatedReuse,
|
|
81
|
+
UserDeletedReuse,
|
|
82
|
+
UserFollowedOrganization,
|
|
83
|
+
]:
|
|
84
|
+
count = 0
|
|
85
|
+
activities = (
|
|
86
|
+
object_related_activity.objects(organization__exists=True)
|
|
87
|
+
.no_dereference() # We use no_dereference in query to prevent DBref DoesNotExist errors
|
|
88
|
+
.no_cache()
|
|
89
|
+
.timeout(False)
|
|
90
|
+
)
|
|
91
|
+
log.info(
|
|
92
|
+
f"{datetime.utcnow()}: Processing {activities.count()} {object_related_activity} activities..."
|
|
93
|
+
)
|
|
94
|
+
for act in activities:
|
|
95
|
+
count += UserUpdatedOrganization.objects(
|
|
96
|
+
actor=act.actor.id,
|
|
97
|
+
related_to=act.organization,
|
|
98
|
+
created_at__gte=act.created_at - timedelta(seconds=1),
|
|
99
|
+
created_at__lte=act.created_at + timedelta(seconds=1),
|
|
100
|
+
).delete()
|
|
101
|
+
log.info(f"{datetime.utcnow()}: Deleted {count} UserUpdatedOrganization activities")
|
udata/mongo/datetime_fields.py
CHANGED
udata/settings.py
CHANGED
|
@@ -574,6 +574,10 @@ class Defaults(object):
|
|
|
574
574
|
###########################################################################
|
|
575
575
|
MAX_RESOURCES_IN_JSON_LD = 20
|
|
576
576
|
|
|
577
|
+
# Metrics settings
|
|
578
|
+
###########################################################################
|
|
579
|
+
METRICS_API = None
|
|
580
|
+
|
|
577
581
|
|
|
578
582
|
class Testing(object):
|
|
579
583
|
"""Sane values for testing. Should be applied as override"""
|
|
@@ -7,7 +7,7 @@ from udata.core.dataset.factories import DatasetFactory
|
|
|
7
7
|
from udata.core.dataset.models import Dataset
|
|
8
8
|
from udata.core.reuse.factories import ReuseFactory
|
|
9
9
|
from udata.core.reuse.models import Reuse
|
|
10
|
-
from udata.core.user.factories import UserFactory
|
|
10
|
+
from udata.core.user.factories import AdminFactory, UserFactory
|
|
11
11
|
from udata.mongo import db
|
|
12
12
|
from udata.tests.helpers import assert200, assert400
|
|
13
13
|
|
|
@@ -67,3 +67,31 @@ class ActivityAPITest:
|
|
|
67
67
|
assert200(response)
|
|
68
68
|
len(response.json["data"]) == 1
|
|
69
69
|
assert response.json["data"][0]["related_to"] == reuse.title
|
|
70
|
+
|
|
71
|
+
def test_activity_api_list_with_private(self, api) -> None:
|
|
72
|
+
"""It should fetch an activity list from the API"""
|
|
73
|
+
activities: list[Activity] = [
|
|
74
|
+
FakeDatasetActivity.objects.create(
|
|
75
|
+
actor=UserFactory(), related_to=DatasetFactory(private=True)
|
|
76
|
+
),
|
|
77
|
+
FakeReuseActivity.objects.create(
|
|
78
|
+
actor=UserFactory(), related_to=ReuseFactory(private=True)
|
|
79
|
+
),
|
|
80
|
+
]
|
|
81
|
+
|
|
82
|
+
# Anonymised user won't see activities about private documents
|
|
83
|
+
response: TestResponse = api.get(url_for("api.activity"))
|
|
84
|
+
assert200(response)
|
|
85
|
+
assert len(response.json["data"]) == 0
|
|
86
|
+
|
|
87
|
+
# Lambda user won't see activities about private documents
|
|
88
|
+
api.login()
|
|
89
|
+
response: TestResponse = api.get(url_for("api.activity"))
|
|
90
|
+
assert200(response)
|
|
91
|
+
assert len(response.json["data"]) == 0
|
|
92
|
+
|
|
93
|
+
# Sysadmin user will see activities about private documents
|
|
94
|
+
api.login(AdminFactory())
|
|
95
|
+
response: TestResponse = api.get(url_for("api.activity"))
|
|
96
|
+
assert200(response)
|
|
97
|
+
assert len(response.json["data"]) == len(activities)
|