udata 9.1.4__py2.py3-none-any.whl → 9.1.4.dev30973__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of udata might be problematic. Click here for more details.
- tasks/__init__.py +2 -2
- udata/__init__.py +1 -1
- udata/api/__init__.py +3 -2
- udata/api/commands.py +1 -0
- udata/api/fields.py +1 -22
- udata/api_fields.py +37 -140
- udata/app.py +1 -1
- udata/auth/__init__.py +12 -8
- udata/commands/db.py +3 -3
- udata/commands/dcat.py +1 -1
- udata/commands/fixtures.py +40 -60
- udata/commands/purge.py +1 -2
- udata/commands/tests/test_fixtures.py +11 -44
- udata/core/activity/api.py +1 -14
- udata/core/activity/tasks.py +1 -1
- udata/core/badges/models.py +2 -6
- udata/core/contact_point/api.py +3 -1
- udata/core/dataservices/api.py +1 -37
- udata/core/dataservices/models.py +0 -38
- udata/core/dataservices/tasks.py +1 -1
- udata/core/dataset/events.py +1 -4
- udata/core/dataset/forms.py +2 -0
- udata/core/dataset/models.py +10 -12
- udata/core/dataset/rdf.py +1 -1
- udata/core/discussions/api.py +1 -1
- udata/core/discussions/models.py +2 -2
- udata/core/discussions/tasks.py +1 -1
- udata/core/metrics/models.py +1 -4
- udata/core/organization/api.py +7 -11
- udata/core/organization/api_fields.py +4 -10
- udata/core/organization/apiv2.py +1 -1
- udata/core/organization/csv.py +0 -1
- udata/core/organization/rdf.py +1 -4
- udata/core/owned.py +2 -4
- udata/core/post/api.py +2 -2
- udata/core/reuse/api.py +25 -32
- udata/core/reuse/api_fields.py +101 -2
- udata/core/reuse/apiv2.py +4 -4
- udata/core/reuse/forms.py +45 -0
- udata/core/reuse/models.py +16 -98
- udata/core/site/api.py +29 -3
- udata/core/spatial/commands.py +3 -3
- udata/core/spatial/factories.py +1 -1
- udata/core/spatial/forms.py +1 -1
- udata/core/spatial/models.py +2 -2
- udata/core/spatial/tests/test_models.py +1 -1
- udata/core/spatial/translations.py +1 -3
- udata/core/topic/api.py +2 -2
- udata/core/topic/apiv2.py +2 -1
- udata/core/user/api.py +8 -28
- udata/core/user/metrics.py +1 -1
- udata/cors.py +4 -4
- udata/features/transfer/api.py +2 -1
- udata/harvest/actions.py +1 -1
- udata/harvest/backends/__init__.py +1 -1
- udata/harvest/tasks.py +1 -0
- udata/harvest/tests/factories.py +2 -0
- udata/harvest/tests/test_base_backend.py +1 -0
- udata/harvest/tests/test_dcat_backend.py +17 -16
- udata/migrations/2020-07-24-remove-s-from-scope-oauth.py +1 -1
- udata/migrations/2021-07-05-remove-unused-badges.py +1 -0
- udata/migrations/2023-02-08-rename-internal-dates.py +2 -0
- udata/migrations/2024-06-11-fix-reuse-datasets-references.py +1 -0
- udata/mongo/datetime_fields.py +4 -11
- udata/mongo/document.py +0 -2
- udata/mongo/taglist_field.py +0 -26
- udata/search/commands.py +1 -1
- udata/search/query.py +1 -1
- udata/settings.py +0 -1
- udata/static/admin.js +36 -36
- udata/static/admin.js.map +1 -1
- udata/static/chunks/{12.576e63b7a990f8eab784.js → 12.5b900cac4417e10ef3a0.js} +2 -2
- udata/static/chunks/12.5b900cac4417e10ef3a0.js.map +1 -0
- udata/static/chunks/{28.1ef31a46255dc2bf56d1.js → 28.1759a7f57d526e6db574.js} +2 -2
- udata/static/chunks/28.1759a7f57d526e6db574.js.map +1 -0
- udata/static/common.js +1 -1
- udata/static/common.js.map +1 -1
- udata/tests/api/test_base_api.py +1 -1
- udata/tests/api/test_contact_points.py +4 -4
- udata/tests/api/test_dataservices_api.py +0 -59
- udata/tests/api/test_datasets_api.py +10 -10
- udata/tests/api/test_organizations_api.py +39 -39
- udata/tests/api/test_reuses_api.py +0 -49
- udata/tests/api/test_tags_api.py +4 -4
- udata/tests/api/test_transfer_api.py +1 -1
- udata/tests/api/test_user_api.py +0 -11
- udata/tests/apiv2/test_datasets.py +4 -4
- udata/tests/dataset/test_dataset_events.py +0 -28
- udata/tests/dataset/test_dataset_model.py +3 -3
- udata/tests/frontend/__init__.py +2 -0
- udata/tests/frontend/test_auth.py +1 -0
- udata/tests/organization/test_csv_adapter.py +2 -0
- udata/tests/organization/test_notifications.py +3 -3
- udata/tests/organization/test_organization_rdf.py +6 -31
- udata/tests/reuse/test_reuse_model.py +1 -0
- udata/tests/site/test_site_rdf.py +3 -1
- udata/tests/test_cors.py +3 -0
- udata/tests/test_owned.py +4 -4
- udata/tests/test_routing.py +1 -1
- udata/tests/test_tags.py +1 -1
- udata/tests/test_transfer.py +2 -1
- udata/tests/workers/test_jobs_commands.py +1 -1
- udata/utils.py +0 -12
- {udata-9.1.4.dist-info → udata-9.1.4.dev30973.dist-info}/METADATA +4 -16
- {udata-9.1.4.dist-info → udata-9.1.4.dev30973.dist-info}/RECORD +109 -109
- udata/static/chunks/12.576e63b7a990f8eab784.js.map +0 -1
- udata/static/chunks/28.1ef31a46255dc2bf56d1.js.map +0 -1
- udata/tests/api/test_activities_api.py +0 -69
- {udata-9.1.4.dist-info → udata-9.1.4.dev30973.dist-info}/LICENSE +0 -0
- {udata-9.1.4.dist-info → udata-9.1.4.dev30973.dist-info}/WHEEL +0 -0
- {udata-9.1.4.dist-info → udata-9.1.4.dev30973.dist-info}/entry_points.txt +0 -0
- {udata-9.1.4.dist-info → udata-9.1.4.dev30973.dist-info}/top_level.txt +0 -0
udata/commands/purge.py
CHANGED
|
@@ -3,9 +3,8 @@ import logging
|
|
|
3
3
|
import click
|
|
4
4
|
|
|
5
5
|
from udata.commands import cli, success
|
|
6
|
-
|
|
7
|
-
from udata.core.dataset.tasks import purge_datasets # isort: skip
|
|
8
6
|
from udata.core.dataservices.tasks import purge_dataservices
|
|
7
|
+
from udata.core.dataset.tasks import purge_datasets
|
|
9
8
|
from udata.core.organization.tasks import purge_organizations
|
|
10
9
|
from udata.core.reuse.tasks import purge_reuses
|
|
11
10
|
|
|
@@ -1,7 +1,10 @@
|
|
|
1
|
+
import json
|
|
1
2
|
from tempfile import NamedTemporaryFile
|
|
2
3
|
|
|
3
4
|
import pytest
|
|
4
5
|
import requests
|
|
6
|
+
import werkzeug.test
|
|
7
|
+
from pytest_mock import MockerFixture
|
|
5
8
|
from werkzeug.wrappers.response import Response
|
|
6
9
|
|
|
7
10
|
import udata.commands.fixtures
|
|
@@ -14,7 +17,7 @@ from udata.core.dataset.factories import (
|
|
|
14
17
|
)
|
|
15
18
|
from udata.core.discussions.factories import DiscussionFactory, MessageDiscussionFactory
|
|
16
19
|
from udata.core.organization.factories import OrganizationFactory
|
|
17
|
-
from udata.core.organization.models import Member
|
|
20
|
+
from udata.core.organization.models import Member, Organization
|
|
18
21
|
from udata.core.reuse.factories import ReuseFactory
|
|
19
22
|
from udata.core.user.factories import UserFactory
|
|
20
23
|
|
|
@@ -27,28 +30,21 @@ class FixturesTest:
|
|
|
27
30
|
"""Test generating fixtures from the current env, then importing them back."""
|
|
28
31
|
assert models.Dataset.objects.count() == 0 # Start with a clean slate.
|
|
29
32
|
user = UserFactory()
|
|
30
|
-
|
|
31
|
-
org = OrganizationFactory(
|
|
32
|
-
members=[Member(user=user, role="editor"), Member(user=admin, role="admin")]
|
|
33
|
-
)
|
|
33
|
+
org = OrganizationFactory(**{}, members=[Member(user=user)])
|
|
34
34
|
# Set the same slug we're 'exporting' from the FIXTURE_DATASET_SLUG config, see the
|
|
35
35
|
# @pytest.mark.options above.
|
|
36
|
-
dataset = DatasetFactory(slug="some-test-dataset-slug", organization=org)
|
|
37
|
-
res = ResourceFactory()
|
|
36
|
+
dataset = DatasetFactory(**{}, slug="some-test-dataset-slug", organization=org)
|
|
37
|
+
res = ResourceFactory(**{})
|
|
38
38
|
dataset.add_resource(res)
|
|
39
|
-
ReuseFactory(datasets=[dataset], owner=user)
|
|
40
|
-
CommunityResourceFactory(dataset=dataset, owner=user)
|
|
39
|
+
ReuseFactory(**{}, datasets=[dataset], owner=user)
|
|
40
|
+
CommunityResourceFactory(**{}, dataset=dataset, owner=user)
|
|
41
41
|
DiscussionFactory(
|
|
42
42
|
**{},
|
|
43
43
|
subject=dataset,
|
|
44
44
|
user=user,
|
|
45
|
-
discussion=[
|
|
46
|
-
MessageDiscussionFactory(posted_by=user),
|
|
47
|
-
MessageDiscussionFactory(posted_by=admin),
|
|
48
|
-
],
|
|
49
|
-
closed_by=admin,
|
|
45
|
+
discussion=[MessageDiscussionFactory(**{}, posted_by=user)],
|
|
50
46
|
)
|
|
51
|
-
DataserviceFactory(datasets=[dataset]
|
|
47
|
+
DataserviceFactory(**{}, datasets=[dataset])
|
|
52
48
|
|
|
53
49
|
with NamedTemporaryFile(mode="w+", delete=True) as fixtures_fd:
|
|
54
50
|
# Get the fixtures from the local instance.
|
|
@@ -58,43 +54,14 @@ class FixturesTest:
|
|
|
58
54
|
fixtures_fd.flush()
|
|
59
55
|
assert "Fixtures saved to file " in result.output
|
|
60
56
|
|
|
61
|
-
# Delete everything, so we can make sure the objects are imported.
|
|
62
|
-
models.Organization.drop_collection()
|
|
63
|
-
models.Dataset.drop_collection()
|
|
64
|
-
models.Discussion.drop_collection()
|
|
65
|
-
models.CommunityResource.drop_collection()
|
|
66
|
-
models.User.drop_collection()
|
|
67
|
-
models.Dataservice.drop_collection()
|
|
68
|
-
|
|
69
|
-
assert models.Organization.objects(slug=org.slug).count() == 0
|
|
70
|
-
assert models.Dataset.objects.count() == 0
|
|
71
|
-
assert models.Discussion.objects.count() == 0
|
|
72
|
-
assert models.CommunityResource.objects.count() == 0
|
|
73
|
-
assert models.User.objects.count() == 0
|
|
74
|
-
assert models.Dataservice.objects.count() == 0
|
|
75
|
-
|
|
76
57
|
# Then load them in the database to make sure they're correct.
|
|
77
58
|
result = cli("import-fixtures", fixtures_fd.name)
|
|
78
59
|
assert models.Organization.objects(slug=org.slug).count() > 0
|
|
79
|
-
result_org = models.Organization.objects.get(slug=org.slug)
|
|
80
|
-
assert result_org.members[0].user.id == user.id
|
|
81
|
-
assert result_org.members[0].role == "editor"
|
|
82
|
-
assert result_org.members[1].user.id == admin.id
|
|
83
|
-
assert result_org.members[1].role == "admin"
|
|
84
60
|
assert models.Dataset.objects.count() > 0
|
|
85
61
|
assert models.Discussion.objects.count() > 0
|
|
86
|
-
result_discussion = models.Discussion.objects.first()
|
|
87
|
-
assert result_discussion.user.id == user.id
|
|
88
|
-
assert result_discussion.closed_by.id == admin.id
|
|
89
|
-
assert len(result_discussion.discussion) == 2
|
|
90
|
-
assert result_discussion.discussion[0].posted_by.id == user.id
|
|
91
|
-
assert result_discussion.discussion[1].posted_by.id == admin.id
|
|
92
62
|
assert models.CommunityResource.objects.count() > 0
|
|
93
63
|
assert models.User.objects.count() > 0
|
|
94
64
|
assert models.Dataservice.objects.count() > 0
|
|
95
|
-
# Make sure we also import the dataservice organization
|
|
96
|
-
result_dataservice = models.Dataservice.objects.first()
|
|
97
|
-
assert result_dataservice.organization == org
|
|
98
65
|
|
|
99
66
|
def test_import_fixtures_from_default_file(self, cli):
|
|
100
67
|
"""Test importing fixtures from udata.commands.fixture.DEFAULT_FIXTURE_FILE."""
|
udata/core/activity/api.py
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import logging
|
|
2
2
|
|
|
3
|
-
from bson import ObjectId
|
|
4
3
|
from mongoengine.errors import DoesNotExist
|
|
5
4
|
|
|
6
5
|
from udata.api import API, api, fields
|
|
@@ -62,19 +61,13 @@ activity_parser.add_argument(
|
|
|
62
61
|
help="Filter activities for that particular organization",
|
|
63
62
|
location="args",
|
|
64
63
|
)
|
|
65
|
-
activity_parser.add_argument(
|
|
66
|
-
"related_to",
|
|
67
|
-
type=str,
|
|
68
|
-
help="Filter activities for that particular object id (ex : reuse, dataset, etc.)",
|
|
69
|
-
location="args",
|
|
70
|
-
)
|
|
71
64
|
|
|
72
65
|
|
|
73
66
|
@api.route("/activity", endpoint="activity")
|
|
74
67
|
class SiteActivityAPI(API):
|
|
75
68
|
@api.doc("activity")
|
|
76
69
|
@api.expect(activity_parser)
|
|
77
|
-
@api.
|
|
70
|
+
@api.marshal_list_with(activity_page_fields)
|
|
78
71
|
def get(self):
|
|
79
72
|
"""Fetch site activity, optionally filtered by user of org."""
|
|
80
73
|
args = activity_parser.parse_args()
|
|
@@ -86,12 +79,6 @@ class SiteActivityAPI(API):
|
|
|
86
79
|
if args["user"]:
|
|
87
80
|
qs = qs(actor=args["user"])
|
|
88
81
|
|
|
89
|
-
if args["related_to"]:
|
|
90
|
-
if not ObjectId.is_valid(args["related_to"]):
|
|
91
|
-
api.abort(400, "`related_to` arg must be an identifier")
|
|
92
|
-
|
|
93
|
-
qs = qs(related_to=args["related_to"])
|
|
94
|
-
|
|
95
82
|
qs = qs.order_by("-created_at")
|
|
96
83
|
qs = qs.paginate(args["page"], args["page_size"])
|
|
97
84
|
|
udata/core/activity/tasks.py
CHANGED
udata/core/badges/models.py
CHANGED
|
@@ -5,9 +5,9 @@ from mongoengine.signals import post_save
|
|
|
5
5
|
|
|
6
6
|
from udata.api_fields import field
|
|
7
7
|
from udata.auth import current_user
|
|
8
|
-
from udata.core.badges.fields import badge_fields
|
|
9
8
|
from udata.mongo import db
|
|
10
9
|
|
|
10
|
+
from .fields import badge_fields
|
|
11
11
|
from .signals import on_badge_added, on_badge_removed
|
|
12
12
|
|
|
13
13
|
log = logging.getLogger(__name__)
|
|
@@ -42,11 +42,7 @@ class BadgesList(db.EmbeddedDocumentListField):
|
|
|
42
42
|
|
|
43
43
|
|
|
44
44
|
class BadgeMixin(object):
|
|
45
|
-
badges =
|
|
46
|
-
BadgesList(),
|
|
47
|
-
readonly=True,
|
|
48
|
-
inner_field_info={"nested_fields": badge_fields},
|
|
49
|
-
)
|
|
45
|
+
badges = BadgesList()
|
|
50
46
|
|
|
51
47
|
def get_badge(self, kind):
|
|
52
48
|
"""Get a badge given its kind if present"""
|
udata/core/contact_point/api.py
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
from udata.api import API, api
|
|
2
2
|
from udata.api.parsers import ModelApiParser
|
|
3
|
+
from udata.auth import admin_permission
|
|
3
4
|
|
|
4
|
-
from .api_fields import contact_point_fields
|
|
5
|
+
from .api_fields import contact_point_fields, contact_point_page_fields
|
|
5
6
|
from .forms import ContactPointForm
|
|
7
|
+
from .models import ContactPoint
|
|
6
8
|
|
|
7
9
|
|
|
8
10
|
class ContactPointApiParser(ModelApiParser):
|
udata/core/dataservices/api.py
CHANGED
|
@@ -1,23 +1,18 @@
|
|
|
1
1
|
from datetime import datetime
|
|
2
2
|
|
|
3
3
|
import mongoengine
|
|
4
|
-
from flask import
|
|
4
|
+
from flask import request
|
|
5
5
|
from flask_login import current_user
|
|
6
6
|
|
|
7
7
|
from udata.api import API, api
|
|
8
8
|
from udata.api_fields import patch
|
|
9
9
|
from udata.core.dataset.permissions import OwnablePermission
|
|
10
10
|
from udata.core.followers.api import FollowAPI
|
|
11
|
-
from udata.rdf import RDF_EXTENSIONS, graph_response, negociate_content
|
|
12
11
|
|
|
13
12
|
from .models import Dataservice
|
|
14
|
-
from .permissions import DataserviceEditPermission
|
|
15
|
-
from .rdf import dataservice_to_rdf
|
|
16
13
|
|
|
17
14
|
ns = api.namespace("dataservices", "Dataservices related operations (beta)")
|
|
18
15
|
|
|
19
|
-
common_doc = {"params": {"dataservice": "The dataservice ID or slug"}}
|
|
20
|
-
|
|
21
16
|
|
|
22
17
|
@ns.route("/", endpoint="dataservices")
|
|
23
18
|
class DataservicesAPI(API):
|
|
@@ -92,37 +87,6 @@ class DataserviceAPI(API):
|
|
|
92
87
|
return "", 204
|
|
93
88
|
|
|
94
89
|
|
|
95
|
-
@ns.route("/<dataservice:dataservice>/rdf", endpoint="dataservice_rdf", doc=common_doc)
|
|
96
|
-
@api.response(404, "Dataservice not found")
|
|
97
|
-
@api.response(410, "Dataservice has been deleted")
|
|
98
|
-
class DataserviceRdfAPI(API):
|
|
99
|
-
@api.doc("rdf_dataservice")
|
|
100
|
-
def get(self, dataservice):
|
|
101
|
-
format = RDF_EXTENSIONS[negociate_content()]
|
|
102
|
-
url = url_for("api.dataservice_rdf_format", dataservice=dataservice.id, format=format)
|
|
103
|
-
return redirect(url)
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
@ns.route(
|
|
107
|
-
"/<dataservice:dataservice>/rdf.<format>", endpoint="dataservice_rdf_format", doc=common_doc
|
|
108
|
-
)
|
|
109
|
-
@api.response(404, "Dataservice not found")
|
|
110
|
-
@api.response(410, "Dataservice has been deleted")
|
|
111
|
-
class DataserviceRdfFormatAPI(API):
|
|
112
|
-
@api.doc("rdf_dataservice_format")
|
|
113
|
-
def get(self, dataservice, format):
|
|
114
|
-
if not DataserviceEditPermission(dataservice).can():
|
|
115
|
-
if dataservice.private:
|
|
116
|
-
api.abort(404)
|
|
117
|
-
elif dataservice.deleted_at:
|
|
118
|
-
api.abort(410)
|
|
119
|
-
|
|
120
|
-
resource = dataservice_to_rdf(dataservice)
|
|
121
|
-
# bypass flask-restplus make_response, since graph_response
|
|
122
|
-
# is handling the content negociation directly
|
|
123
|
-
return make_response(*graph_response(resource, format))
|
|
124
|
-
|
|
125
|
-
|
|
126
90
|
@ns.route("/<id>/followers/", endpoint="dataservice_followers")
|
|
127
91
|
@ns.doc(
|
|
128
92
|
get={"id": "list_dataservice_followers"},
|
|
@@ -1,7 +1,5 @@
|
|
|
1
1
|
from datetime import datetime
|
|
2
2
|
|
|
3
|
-
from mongoengine import Q
|
|
4
|
-
|
|
5
3
|
import udata.core.contact_point.api_fields as contact_api_fields
|
|
6
4
|
import udata.core.dataset.api_fields as datasets_api_fields
|
|
7
5
|
from udata.api_fields import field, function_field, generate_fields
|
|
@@ -31,42 +29,6 @@ class DataserviceQuerySet(OwnedQuerySet):
|
|
|
31
29
|
def hidden(self):
|
|
32
30
|
return self(db.Q(private=True) | db.Q(deleted_at__ne=None) | db.Q(archived_at__ne=None))
|
|
33
31
|
|
|
34
|
-
def filter_by_dataset_pagination(self, datasets: list[Dataset], page: int):
|
|
35
|
-
"""Paginate the dataservices on the datasets provided.
|
|
36
|
-
|
|
37
|
-
This is a workaround, used (at least) in the catalogs for sites and organizations.
|
|
38
|
-
We paginate those kinda weirdly, on their datasets. So a given organization or site
|
|
39
|
-
catalog will only list a `page_size` number of datasets, but we'd still want to display
|
|
40
|
-
the site's or org's dataservices.
|
|
41
|
-
We can't "double paginate", so instead:
|
|
42
|
-
- only if it's the first page, list all the dataservices that serve no dataset
|
|
43
|
-
- list all the dataservices that serve the datasets in this page
|
|
44
|
-
"""
|
|
45
|
-
# We need to add Dataservice to the catalog.
|
|
46
|
-
# In the best world, we want:
|
|
47
|
-
# - Keep the correct number of datasets on the page (if the requested page size is 100, we should have 100 datasets)
|
|
48
|
-
# - Have simple MongoDB queries
|
|
49
|
-
# - Do not duplicate the datasets (each dataset is present once in the catalog)
|
|
50
|
-
# - Do not duplicate the dataservices (each dataservice is present once in the catalog)
|
|
51
|
-
# - Every referenced dataset for one dataservices present on the page (hard to do)
|
|
52
|
-
#
|
|
53
|
-
# Multiple solutions are possible but none check all the constraints.
|
|
54
|
-
# The selected one is to put all the dataservices referencing at least one of the dataset on
|
|
55
|
-
# the page at the end of it. It means dataservices could be duplicated (present on multiple pages)
|
|
56
|
-
# and these dataservices may referenced some datasets not present in the current page. It's working
|
|
57
|
-
# if somebody is doing the same thing as us (keeping the list of all the datasets IDs for the entire catalog then
|
|
58
|
-
# listing all dataservices in a second pass)
|
|
59
|
-
# Another option is to do some tricky Mongo requests to order/group datasets by their presence in some dataservices but
|
|
60
|
-
# it could be really hard to do with a n..n relation.
|
|
61
|
-
# Let's keep this solution simple right now and iterate on it in the future.
|
|
62
|
-
dataservices_filter = Q(datasets__in=[d.id for d in datasets])
|
|
63
|
-
|
|
64
|
-
# On the first page, add all dataservices without datasets
|
|
65
|
-
if page == 1:
|
|
66
|
-
dataservices_filter = dataservices_filter | Q(datasets__size=0)
|
|
67
|
-
|
|
68
|
-
return self(dataservices_filter)
|
|
69
|
-
|
|
70
32
|
|
|
71
33
|
@generate_fields()
|
|
72
34
|
class HarvestMetadata(db.EmbeddedDocument):
|
udata/core/dataservices/tasks.py
CHANGED
|
@@ -3,7 +3,7 @@ from celery.utils.log import get_task_logger
|
|
|
3
3
|
from udata.core.dataservices.models import Dataservice
|
|
4
4
|
|
|
5
5
|
# from udata.harvest.models import HarvestJob
|
|
6
|
-
from udata.models import Discussion, Follow, Transfer
|
|
6
|
+
from udata.models import Activity, Discussion, Follow, Transfer
|
|
7
7
|
from udata.tasks import job
|
|
8
8
|
|
|
9
9
|
log = get_task_logger(__name__)
|
udata/core/dataset/events.py
CHANGED
|
@@ -49,10 +49,7 @@ def publish(url, document, resource_id, action):
|
|
|
49
49
|
"dataset_id": str(document.id),
|
|
50
50
|
"document": resource,
|
|
51
51
|
}
|
|
52
|
-
|
|
53
|
-
if current_app.config["RESOURCES_ANALYSER_API_KEY"]:
|
|
54
|
-
headers = {"Authorization": f"Bearer {current_app.config['RESOURCES_ANALYSER_API_KEY']}"}
|
|
55
|
-
r = requests.post(url, json=payload, headers=headers)
|
|
52
|
+
r = requests.post(url, json=payload)
|
|
56
53
|
r.raise_for_status()
|
|
57
54
|
|
|
58
55
|
|
udata/core/dataset/forms.py
CHANGED
udata/core/dataset/models.py
CHANGED
|
@@ -256,8 +256,8 @@ class License(db.Document):
|
|
|
256
256
|
|
|
257
257
|
if license is None:
|
|
258
258
|
# Try to single match `slug` with a low Damerau-Levenshtein distance
|
|
259
|
-
computed = ((
|
|
260
|
-
candidates = [
|
|
259
|
+
computed = ((l, rdlevenshtein(l.slug, slug)) for l in cls.objects)
|
|
260
|
+
candidates = [l for l, d in computed if d <= MAX_DISTANCE]
|
|
261
261
|
# If there is more that one match, we cannot determinate
|
|
262
262
|
# which one is closer to safely choose between candidates
|
|
263
263
|
if len(candidates) == 1:
|
|
@@ -265,10 +265,8 @@ class License(db.Document):
|
|
|
265
265
|
|
|
266
266
|
if license is None:
|
|
267
267
|
# Try to match `title` with a low Damerau-Levenshtein distance
|
|
268
|
-
computed = (
|
|
269
|
-
|
|
270
|
-
)
|
|
271
|
-
candidates = [license_ for license_, d in computed if d <= MAX_DISTANCE]
|
|
268
|
+
computed = ((l, rdlevenshtein(l.title.lower(), text)) for l in cls.objects)
|
|
269
|
+
candidates = [l for l, d in computed if d <= MAX_DISTANCE]
|
|
272
270
|
# If there is more that one match, we cannot determinate
|
|
273
271
|
# which one is closer to safely choose between candidates
|
|
274
272
|
if len(candidates) == 1:
|
|
@@ -277,11 +275,11 @@ class License(db.Document):
|
|
|
277
275
|
if license is None:
|
|
278
276
|
# Try to single match `alternate_titles` with a low Damerau-Levenshtein distance
|
|
279
277
|
computed = (
|
|
280
|
-
(
|
|
281
|
-
for
|
|
282
|
-
for
|
|
278
|
+
(l, rdlevenshtein(cls.slug.slugify(t), slug))
|
|
279
|
+
for l in cls.objects
|
|
280
|
+
for t in l.alternate_titles
|
|
283
281
|
)
|
|
284
|
-
candidates = [
|
|
282
|
+
candidates = [l for l, d in computed if d <= MAX_DISTANCE]
|
|
285
283
|
# If there is more that one license matching, we cannot determinate
|
|
286
284
|
# which one is closer to safely choose between candidates
|
|
287
285
|
if len(set(candidates)) == 1:
|
|
@@ -1000,7 +998,7 @@ class ResourceSchema(object):
|
|
|
1000
998
|
f"Schemas catalog does not exist at {endpoint}"
|
|
1001
999
|
)
|
|
1002
1000
|
response.raise_for_status()
|
|
1003
|
-
except requests.exceptions.RequestException:
|
|
1001
|
+
except requests.exceptions.RequestException as e:
|
|
1004
1002
|
log.exception(f"Error while getting schema catalog from {endpoint}")
|
|
1005
1003
|
schemas = cache.get(cache_key)
|
|
1006
1004
|
else:
|
|
@@ -1008,7 +1006,7 @@ class ResourceSchema(object):
|
|
|
1008
1006
|
cache.set(cache_key, schemas)
|
|
1009
1007
|
# no cached version or no content
|
|
1010
1008
|
if not schemas:
|
|
1011
|
-
log.error("No content found inc. from cache for schema catalog")
|
|
1009
|
+
log.error(f"No content found inc. from cache for schema catalog")
|
|
1012
1010
|
raise SchemasCacheUnavailableException("No content in cache for schema catalog")
|
|
1013
1011
|
|
|
1014
1012
|
return schemas
|
udata/core/dataset/rdf.py
CHANGED
|
@@ -387,7 +387,7 @@ def spatial_from_rdf(graph):
|
|
|
387
387
|
continue
|
|
388
388
|
|
|
389
389
|
if not polygons:
|
|
390
|
-
log.warning("No supported types found in the GeoJSON data.")
|
|
390
|
+
log.warning(f"No supported types found in the GeoJSON data.")
|
|
391
391
|
return None
|
|
392
392
|
|
|
393
393
|
spatial_coverage = SpatialCoverage(
|
udata/core/discussions/api.py
CHANGED
|
@@ -152,7 +152,7 @@ class DiscussionAPI(API):
|
|
|
152
152
|
|
|
153
153
|
@ns.route("/<id>/comments/<int:cidx>/spam", endpoint="discussion_comment_spam")
|
|
154
154
|
@ns.doc(delete={"id": "unspam"})
|
|
155
|
-
class
|
|
155
|
+
class DiscussionSpamAPI(SpamAPIMixin):
|
|
156
156
|
def get_model(self, id, cidx):
|
|
157
157
|
discussion = Discussion.objects.get_or_404(id=id_or_404(id))
|
|
158
158
|
if len(discussion.discussion) <= cidx:
|
udata/core/discussions/models.py
CHANGED
|
@@ -20,7 +20,7 @@ class Message(SpamMixin, db.EmbeddedDocument):
|
|
|
20
20
|
return [self.content]
|
|
21
21
|
|
|
22
22
|
def spam_report_message(self, breadcrumb):
|
|
23
|
-
message = "Spam potentiel dans le message"
|
|
23
|
+
message = f"Spam potentiel dans le message"
|
|
24
24
|
if self.posted_by:
|
|
25
25
|
message += f" de [{self.posted_by.fullname}]({self.posted_by.external_url})"
|
|
26
26
|
|
|
@@ -34,7 +34,7 @@ class Message(SpamMixin, db.EmbeddedDocument):
|
|
|
34
34
|
discussion = breadcrumb[0]
|
|
35
35
|
if not isinstance(discussion, Discussion):
|
|
36
36
|
log.warning(
|
|
37
|
-
"`spam_report_message` called on message with a breadcrumb not containing a Discussion at index 0.",
|
|
37
|
+
f"`spam_report_message` called on message with a breadcrumb not containing a Discussion at index 0.",
|
|
38
38
|
extra={"breadcrumb": breadcrumb},
|
|
39
39
|
)
|
|
40
40
|
return message
|
udata/core/discussions/tasks.py
CHANGED
|
@@ -5,7 +5,7 @@ from udata.core.reuse.models import Reuse
|
|
|
5
5
|
from udata.i18n import lazy_gettext as _
|
|
6
6
|
from udata.tasks import connect, get_logger
|
|
7
7
|
|
|
8
|
-
from .models import Discussion
|
|
8
|
+
from .models import Discussion, Message
|
|
9
9
|
from .signals import on_discussion_closed, on_new_discussion, on_new_discussion_comment
|
|
10
10
|
|
|
11
11
|
log = get_logger(__name__)
|
udata/core/metrics/models.py
CHANGED
udata/core/organization/api.py
CHANGED
|
@@ -8,22 +8,19 @@ from udata.api.parsers import ModelApiParser
|
|
|
8
8
|
from udata.auth import admin_permission, current_user
|
|
9
9
|
from udata.core.badges import api as badges_api
|
|
10
10
|
from udata.core.badges.fields import badge_fields
|
|
11
|
-
from udata.core.contact_point.api import ContactPointApiParser
|
|
12
|
-
from udata.core.contact_point.api_fields import contact_point_page_fields
|
|
13
|
-
from udata.core.dataservices.models import Dataservice
|
|
14
11
|
from udata.core.dataset.api import DatasetApiParser
|
|
15
12
|
from udata.core.dataset.api_fields import dataset_page_fields
|
|
16
13
|
from udata.core.dataset.models import Dataset
|
|
17
14
|
from udata.core.discussions.api import discussion_fields
|
|
18
15
|
from udata.core.discussions.models import Discussion
|
|
19
16
|
from udata.core.followers.api import FollowAPI
|
|
17
|
+
from udata.core.reuse.api_fields import reuse_fields
|
|
20
18
|
from udata.core.reuse.models import Reuse
|
|
21
19
|
from udata.core.storages.api import (
|
|
22
20
|
image_parser,
|
|
23
21
|
parse_uploaded_image,
|
|
24
22
|
uploaded_image_fields,
|
|
25
23
|
)
|
|
26
|
-
from udata.models import ContactPoint
|
|
27
24
|
from udata.rdf import RDF_EXTENSIONS, graph_response, negociate_content
|
|
28
25
|
from udata.utils import multi_to_dict
|
|
29
26
|
|
|
@@ -176,12 +173,7 @@ class OrganizationRdfFormatAPI(API):
|
|
|
176
173
|
page = int(params.get("page", 1))
|
|
177
174
|
page_size = int(params.get("page_size", 100))
|
|
178
175
|
datasets = Dataset.objects(organization=org).visible().paginate(page, page_size)
|
|
179
|
-
|
|
180
|
-
Dataservice.objects(organization=org)
|
|
181
|
-
.visible()
|
|
182
|
-
.filter_by_dataset_pagination(datasets, page)
|
|
183
|
-
)
|
|
184
|
-
catalog = build_org_catalog(org, datasets, dataservices, format=format)
|
|
176
|
+
catalog = build_org_catalog(org, datasets, format=format)
|
|
185
177
|
# bypass flask-restplus make_response, since graph_response
|
|
186
178
|
# is handling the content negociation directly
|
|
187
179
|
return make_response(*graph_response(catalog, format))
|
|
@@ -215,6 +207,10 @@ class OrganizationBadgeAPI(API):
|
|
|
215
207
|
return badges_api.remove(org, badge_kind)
|
|
216
208
|
|
|
217
209
|
|
|
210
|
+
from udata.core.contact_point.api import ContactPointApiParser
|
|
211
|
+
from udata.core.contact_point.api_fields import contact_point_page_fields
|
|
212
|
+
from udata.models import ContactPoint
|
|
213
|
+
|
|
218
214
|
contact_point_parser = ContactPointApiParser()
|
|
219
215
|
|
|
220
216
|
|
|
@@ -467,7 +463,7 @@ class OrgDatasetsAPI(API):
|
|
|
467
463
|
@ns.route("/<org:org>/reuses/", endpoint="org_reuses")
|
|
468
464
|
class OrgReusesAPI(API):
|
|
469
465
|
@api.doc("list_organization_reuses")
|
|
470
|
-
@api.marshal_list_with(
|
|
466
|
+
@api.marshal_list_with(reuse_fields)
|
|
471
467
|
def get(self, org):
|
|
472
468
|
"""List organization reuses (including private ones when member)"""
|
|
473
469
|
qs = Reuse.objects.owned_by(org)
|
|
@@ -41,11 +41,10 @@ org_ref_fields = api.inherit(
|
|
|
41
41
|
},
|
|
42
42
|
)
|
|
43
43
|
|
|
44
|
-
|
|
45
|
-
from udata.core.user.api_fields import user_ref_fields # noqa
|
|
44
|
+
from udata.core.user.api_fields import user_ref_fields # noqa: required
|
|
46
45
|
|
|
47
46
|
|
|
48
|
-
def
|
|
47
|
+
def check_can_access_email():
|
|
49
48
|
# This endpoint is secure, only organization member has access.
|
|
50
49
|
if request.endpoint == "api.request_membership":
|
|
51
50
|
return True
|
|
@@ -65,13 +64,8 @@ member_user_with_email_fields = api.inherit(
|
|
|
65
64
|
user_ref_fields,
|
|
66
65
|
{
|
|
67
66
|
"email": fields.Raw(
|
|
68
|
-
attribute=lambda o: o.email if
|
|
69
|
-
description="The user email (only present on show organization endpoint if the current user
|
|
70
|
-
readonly=True,
|
|
71
|
-
),
|
|
72
|
-
"last_login_at": fields.Raw(
|
|
73
|
-
attribute=lambda o: o.last_login_at if check_can_access_user_private_info() else None,
|
|
74
|
-
description="The user last connection date (only present on show organization endpoint if the current user is member of the organization: admin or editor)",
|
|
67
|
+
attribute=lambda o: o.email if check_can_access_email() else None,
|
|
68
|
+
description="The user email (only present on show organization endpoint if the current user has edit permission on the org)",
|
|
75
69
|
readonly=True,
|
|
76
70
|
),
|
|
77
71
|
},
|
udata/core/organization/apiv2.py
CHANGED
|
@@ -7,7 +7,7 @@ from udata.core.contact_point.api_fields import contact_point_fields
|
|
|
7
7
|
from udata.utils import multi_to_dict
|
|
8
8
|
|
|
9
9
|
from .api_fields import member_fields, org_fields, org_page_fields
|
|
10
|
-
from .permissions import EditOrganizationPermission
|
|
10
|
+
from .permissions import EditOrganizationPermission, OrganizationPrivatePermission
|
|
11
11
|
from .search import OrganizationSearch
|
|
12
12
|
|
|
13
13
|
apiv2.inherit("OrganizationPage", org_page_fields)
|
udata/core/organization/csv.py
CHANGED
udata/core/organization/rdf.py
CHANGED
|
@@ -7,7 +7,6 @@ from flask import url_for
|
|
|
7
7
|
from rdflib import BNode, Graph, Literal, URIRef
|
|
8
8
|
from rdflib.namespace import FOAF, RDF, RDFS
|
|
9
9
|
|
|
10
|
-
from udata.core.dataservices.rdf import dataservice_to_rdf
|
|
11
10
|
from udata.core.dataset.rdf import dataset_to_rdf
|
|
12
11
|
from udata.rdf import DCAT, DCT, namespace_manager, paginate_catalog
|
|
13
12
|
from udata.uris import endpoint_for
|
|
@@ -36,7 +35,7 @@ def organization_to_rdf(org, graph=None):
|
|
|
36
35
|
return o
|
|
37
36
|
|
|
38
37
|
|
|
39
|
-
def build_org_catalog(org, datasets,
|
|
38
|
+
def build_org_catalog(org, datasets, format=None):
|
|
40
39
|
graph = Graph(namespace_manager=namespace_manager)
|
|
41
40
|
org_catalog_url = url_for("api.organization_rdf", org=org.id, _external=True)
|
|
42
41
|
|
|
@@ -48,8 +47,6 @@ def build_org_catalog(org, datasets, dataservices, format=None):
|
|
|
48
47
|
|
|
49
48
|
for dataset in datasets:
|
|
50
49
|
catalog.add(DCAT.dataset, dataset_to_rdf(dataset, graph))
|
|
51
|
-
for dataservice in dataservices:
|
|
52
|
-
catalog.add(DCAT.dataservice, dataservice_to_rdf(dataservice, graph))
|
|
53
50
|
|
|
54
51
|
values = {"org": org.id}
|
|
55
52
|
|
udata/core/owned.py
CHANGED
|
@@ -32,7 +32,7 @@ def check_owner_is_current_user(owner):
|
|
|
32
32
|
current_user.is_authenticated
|
|
33
33
|
and owner
|
|
34
34
|
and not admin_permission
|
|
35
|
-
and current_user.id != owner
|
|
35
|
+
and current_user.id != owner
|
|
36
36
|
):
|
|
37
37
|
raise FieldValidationError(_("You can only set yourself as owner"), field="owner")
|
|
38
38
|
|
|
@@ -41,7 +41,7 @@ def check_organization_is_valid_for_current_user(organization):
|
|
|
41
41
|
from udata.auth import current_user
|
|
42
42
|
from udata.models import Organization
|
|
43
43
|
|
|
44
|
-
org = Organization.objects(id=organization
|
|
44
|
+
org = Organization.objects(id=organization).first()
|
|
45
45
|
if org is None:
|
|
46
46
|
raise FieldValidationError(_("Unknown organization"), field="organization")
|
|
47
47
|
|
|
@@ -62,7 +62,6 @@ class Owned(object):
|
|
|
62
62
|
description="Only present if organization is not set. Can only be set to the current authenticated user.",
|
|
63
63
|
check=check_owner_is_current_user,
|
|
64
64
|
allow_null=True,
|
|
65
|
-
filterable={},
|
|
66
65
|
)
|
|
67
66
|
organization = field(
|
|
68
67
|
ReferenceField(Organization, reverse_delete_rule=NULLIFY),
|
|
@@ -70,7 +69,6 @@ class Owned(object):
|
|
|
70
69
|
description="Only present if owner is not set. Can only be set to an organization of the current authenticated user.",
|
|
71
70
|
check=check_organization_is_valid_for_current_user,
|
|
72
71
|
allow_null=True,
|
|
73
|
-
filterable={},
|
|
74
72
|
)
|
|
75
73
|
|
|
76
74
|
on_owner_change = signal("Owned.on_owner_change")
|
udata/core/post/api.py
CHANGED
|
@@ -3,7 +3,7 @@ from datetime import datetime
|
|
|
3
3
|
from udata.api import API, api, fields
|
|
4
4
|
from udata.auth import admin_permission
|
|
5
5
|
from udata.core.dataset.api_fields import dataset_fields
|
|
6
|
-
from udata.core.reuse.
|
|
6
|
+
from udata.core.reuse.api_fields import reuse_fields
|
|
7
7
|
from udata.core.storages.api import (
|
|
8
8
|
image_parser,
|
|
9
9
|
parse_uploaded_image,
|
|
@@ -29,7 +29,7 @@ post_fields = api.model(
|
|
|
29
29
|
"credit_url": fields.String(description="An optional link associated to the credits"),
|
|
30
30
|
"tags": fields.List(fields.String, description="Some keywords to help in search"),
|
|
31
31
|
"datasets": fields.List(fields.Nested(dataset_fields), description="The post datasets"),
|
|
32
|
-
"reuses": fields.List(fields.Nested(
|
|
32
|
+
"reuses": fields.List(fields.Nested(reuse_fields), description="The post reuses"),
|
|
33
33
|
"owner": fields.Nested(
|
|
34
34
|
user_ref_fields, description="The owner user", readonly=True, allow_null=True
|
|
35
35
|
),
|