udata 12.0.2.dev13__py3-none-any.whl → 12.0.2.dev15__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of udata might be problematic. Click here for more details.
- udata/core/dataservices/api.py +2 -4
- udata/core/dataset/api.py +4 -4
- udata/core/reuse/api.py +2 -2
- udata/harvest/tests/dcat/bnodes.xml +17 -1
- udata/harvest/tests/test_dcat_backend.py +25 -0
- udata/rdf.py +29 -2
- udata/settings.py +8 -1
- udata/tests/api/test_dataservices_api.py +8 -4
- udata/tests/api/test_datasets_api.py +25 -7
- udata/tests/api/test_reuses_api.py +7 -4
- udata/utils.py +52 -2
- {udata-12.0.2.dev13.dist-info → udata-12.0.2.dev15.dist-info}/METADATA +1 -1
- {udata-12.0.2.dev13.dist-info → udata-12.0.2.dev15.dist-info}/RECORD +17 -17
- {udata-12.0.2.dev13.dist-info → udata-12.0.2.dev15.dist-info}/WHEEL +0 -0
- {udata-12.0.2.dev13.dist-info → udata-12.0.2.dev15.dist-info}/entry_points.txt +0 -0
- {udata-12.0.2.dev13.dist-info → udata-12.0.2.dev15.dist-info}/licenses/LICENSE +0 -0
- {udata-12.0.2.dev13.dist-info → udata-12.0.2.dev15.dist-info}/top_level.txt +0 -0
udata/core/dataservices/api.py
CHANGED
|
@@ -12,10 +12,10 @@ from udata.auth import admin_permission
|
|
|
12
12
|
from udata.core.dataservices.constants import DATASERVICE_ACCESS_TYPE_RESTRICTED
|
|
13
13
|
from udata.core.dataset.models import Dataset
|
|
14
14
|
from udata.core.followers.api import FollowAPI
|
|
15
|
-
from udata.core.site.models import current_site
|
|
16
15
|
from udata.frontend.markdown import md
|
|
17
16
|
from udata.i18n import gettext as _
|
|
18
17
|
from udata.rdf import RDF_EXTENSIONS, graph_response, negociate_content
|
|
18
|
+
from udata.utils import get_rss_feed_list
|
|
19
19
|
|
|
20
20
|
from .models import Dataservice
|
|
21
21
|
from .rdf import dataservice_to_rdf
|
|
@@ -62,9 +62,7 @@ class DataservicesAtomFeedAPI(API):
|
|
|
62
62
|
_("Latest APIs"), description=None, feed_url=request.url, link=request.url_root
|
|
63
63
|
)
|
|
64
64
|
|
|
65
|
-
dataservices
|
|
66
|
-
Dataservice.objects.visible().order_by("-created_at").limit(current_site.feed_size)
|
|
67
|
-
)
|
|
65
|
+
dataservices = get_rss_feed_list(Dataservice.objects.visible(), "created_at")
|
|
68
66
|
for dataservice in dataservices:
|
|
69
67
|
author_name = None
|
|
70
68
|
author_uri = None
|
udata/core/dataset/api.py
CHANGED
|
@@ -41,13 +41,12 @@ from udata.core.followers.api import FollowAPI
|
|
|
41
41
|
from udata.core.followers.models import Follow
|
|
42
42
|
from udata.core.organization.models import Organization
|
|
43
43
|
from udata.core.reuse.models import Reuse
|
|
44
|
-
from udata.core.site.models import current_site
|
|
45
44
|
from udata.core.storages.api import handle_upload, upload_parser
|
|
46
45
|
from udata.core.topic.models import Topic
|
|
47
46
|
from udata.frontend.markdown import md
|
|
48
47
|
from udata.i18n import gettext as _
|
|
49
48
|
from udata.rdf import RDF_EXTENSIONS, graph_response, negociate_content
|
|
50
|
-
from udata.utils import get_by
|
|
49
|
+
from udata.utils import get_by, get_rss_feed_list
|
|
51
50
|
|
|
52
51
|
from .api_fields import (
|
|
53
52
|
catalog_schema_fields,
|
|
@@ -336,9 +335,10 @@ class DatasetsAtomFeedAPI(API):
|
|
|
336
335
|
link=request.url_root,
|
|
337
336
|
)
|
|
338
337
|
|
|
339
|
-
datasets: list[Dataset] = (
|
|
340
|
-
Dataset.objects.visible()
|
|
338
|
+
datasets: list[Dataset] = get_rss_feed_list(
|
|
339
|
+
Dataset.objects.visible(), "created_at_internal"
|
|
341
340
|
)
|
|
341
|
+
|
|
342
342
|
for dataset in datasets:
|
|
343
343
|
author_name = None
|
|
344
344
|
author_uri = None
|
udata/core/reuse/api.py
CHANGED
|
@@ -24,7 +24,7 @@ from udata.core.storages.api import (
|
|
|
24
24
|
from udata.frontend.markdown import md
|
|
25
25
|
from udata.i18n import gettext as _
|
|
26
26
|
from udata.models import Dataset
|
|
27
|
-
from udata.utils import id_or_404
|
|
27
|
+
from udata.utils import get_rss_feed_list, id_or_404
|
|
28
28
|
|
|
29
29
|
from .api_fields import (
|
|
30
30
|
reuse_suggestion_fields,
|
|
@@ -143,7 +143,7 @@ class ReusesAtomFeedAPI(API):
|
|
|
143
143
|
link=request.url_root,
|
|
144
144
|
)
|
|
145
145
|
|
|
146
|
-
reuses
|
|
146
|
+
reuses = get_rss_feed_list(Reuse.objects.visible(), "created_at")
|
|
147
147
|
for reuse in reuses:
|
|
148
148
|
author_name = None
|
|
149
149
|
author_uri = None
|
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
xmlns:dcterms="http://purl.org/dc/terms/"
|
|
12
12
|
xmlns:vcard="http://www.w3.org/2006/vcard/ns#"
|
|
13
13
|
xmlns:schema="http://schema.org/"
|
|
14
|
+
xmlns:skos="http://www.w3.org/2004/02/skos/core#"
|
|
14
15
|
>
|
|
15
16
|
<dcat:Catalog rdf:about="http://data.test.org/">
|
|
16
17
|
<dcat:dataset>
|
|
@@ -45,7 +46,6 @@
|
|
|
45
46
|
<dcterms:title>Sample DCAT Catalog</dcterms:title>
|
|
46
47
|
<dcat:dataset>
|
|
47
48
|
<dcat:Dataset>
|
|
48
|
-
<dcat:theme>Theme 2</dcat:theme>
|
|
49
49
|
<dcat:contactPoint rdf:resource="http://data.test.org/contacts/1"/>
|
|
50
50
|
<dcat:landingPage>http://data.test.org/datasets/1</dcat:landingPage>
|
|
51
51
|
<dcat:keyword>Tag 3</dcat:keyword>
|
|
@@ -75,6 +75,17 @@
|
|
|
75
75
|
<dcat:distribution rdf:resource="http://data.test.org/datasets/1/resources/1"/>
|
|
76
76
|
<dcterms:issued rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2016-12-14T18:59:02.737480</dcterms:issued>
|
|
77
77
|
<dcterms:identifier>1</dcterms:identifier>
|
|
78
|
+
<dcat:theme>
|
|
79
|
+
<skos:Concept>
|
|
80
|
+
<skos:prefLabel xml:lang="fr">Répartition des espèces</skos:prefLabel>
|
|
81
|
+
<skos:inScheme>
|
|
82
|
+
<skos:ConceptScheme>
|
|
83
|
+
<dcterms:title xml:lang="fr">GEMET - INSPIRE themes, version 1.0</dcterms:title>
|
|
84
|
+
<dcterms:issued rdf:datatype="http://www.w3.org/2001/XMLSchema#date">2018-07-27</dcterms:issued>
|
|
85
|
+
</skos:ConceptScheme>
|
|
86
|
+
</skos:inScheme>
|
|
87
|
+
</skos:Concept>
|
|
88
|
+
</dcat:theme>
|
|
78
89
|
</dcat:Dataset>
|
|
79
90
|
</dcat:dataset>
|
|
80
91
|
<dcat:dataset>
|
|
@@ -110,6 +121,7 @@
|
|
|
110
121
|
</dct:spatial>
|
|
111
122
|
<dcterms:identifier>2</dcterms:identifier>
|
|
112
123
|
<dct:conformsTo rdf:nodeID="Ne0189e93917c4f67a412fc44883322e7"/>
|
|
124
|
+
<dcat:theme rdf:resource="http://bnode.namespace.voc/theme/hy"/>
|
|
113
125
|
</dcat:Dataset>
|
|
114
126
|
</dcat:dataset>
|
|
115
127
|
<dcat:service>
|
|
@@ -185,4 +197,8 @@
|
|
|
185
197
|
<dct:type rdf:resource="http://inspire.ec.europa.eu/glossary/SpatialReferenceSystem"/>
|
|
186
198
|
<dct:title xml:lang="fr">RGF93 / Lambert-93 (EPSG:2154)</dct:title>
|
|
187
199
|
</rdf:Description>
|
|
200
|
+
<skos:Concept rdf:about="http://bnode.namespace.voc/theme/hy">
|
|
201
|
+
<skos:inScheme rdf:resource="http://inspire.ec.europa.eu/theme"/>
|
|
202
|
+
<skos:prefLabel>Hydrographie</skos:prefLabel>
|
|
203
|
+
</skos:Concept>
|
|
188
204
|
</rdf:RDF>
|
|
@@ -445,6 +445,23 @@ class DcatBackendTest:
|
|
|
445
445
|
assert resources_by_title["Resource 3-1"].schema.url is None
|
|
446
446
|
assert resources_by_title["Resource 3-1"].schema.version == "2.2.0"
|
|
447
447
|
|
|
448
|
+
@pytest.mark.options(SCHEMA_CATALOG_URL="https://example.com/schemas")
|
|
449
|
+
def test_harvest_inspire_themese(self, rmock):
|
|
450
|
+
rmock.get("https://example.com/schemas", json=ResourceSchemaMockData.get_mock_data())
|
|
451
|
+
|
|
452
|
+
filename = "bnodes.xml"
|
|
453
|
+
url = mock_dcat(rmock, filename)
|
|
454
|
+
org = OrganizationFactory()
|
|
455
|
+
source = HarvestSourceFactory(backend="dcat", url=url, organization=org)
|
|
456
|
+
|
|
457
|
+
actions.run(source)
|
|
458
|
+
|
|
459
|
+
datasets = {d.harvest.dct_identifier: d for d in Dataset.objects}
|
|
460
|
+
|
|
461
|
+
assert set(datasets["1"].tags).issuperset(set(["repartition-des-especes", "inspire"]))
|
|
462
|
+
assert set(datasets["2"].tags).issuperset(set(["hydrographie", "inspire"]))
|
|
463
|
+
assert "inspire" not in datasets["3"].tags
|
|
464
|
+
|
|
448
465
|
def test_simple_nested_attributes(self, rmock):
|
|
449
466
|
filename = "nested.jsonld"
|
|
450
467
|
url = mock_dcat(rmock, filename)
|
|
@@ -672,6 +689,9 @@ class DcatBackendTest:
|
|
|
672
689
|
assert dataset.temporal_coverage is not None
|
|
673
690
|
assert dataset.temporal_coverage.start == date(2004, 11, 3)
|
|
674
691
|
assert dataset.temporal_coverage.end == date(2005, 3, 30)
|
|
692
|
+
assert set(dataset.tags) == set(
|
|
693
|
+
["inspire", "biodiversity-dynamics"]
|
|
694
|
+
) # The DCAT.theme with rdf:resource don't have labels properly defined
|
|
675
695
|
|
|
676
696
|
def test_sigoreme_xml_catalog(self, rmock):
|
|
677
697
|
LicenseFactory(id="fr-lo", title="Licence ouverte / Open Licence")
|
|
@@ -911,6 +931,7 @@ class CswDcatBackendTest:
|
|
|
911
931
|
"oise",
|
|
912
932
|
"somme",
|
|
913
933
|
"aisne",
|
|
934
|
+
# "inspire", TODO: the geonetwork v4 examples use broken URI as theme resources, check if this is still a problem or not
|
|
914
935
|
]
|
|
915
936
|
)
|
|
916
937
|
assert dataset.harvest.issued_at.date() == date(2017, 1, 1)
|
|
@@ -1085,6 +1106,7 @@ class CswIso19139DcatBackendTest:
|
|
|
1085
1106
|
"donnees-ouvertes",
|
|
1086
1107
|
"plu",
|
|
1087
1108
|
"usage-des-sols",
|
|
1109
|
+
"inspire",
|
|
1088
1110
|
]
|
|
1089
1111
|
)
|
|
1090
1112
|
assert dataset.harvest.issued_at.date() == date(2017, 10, 7)
|
|
@@ -1195,3 +1217,6 @@ class CswIso19139DcatBackendTest:
|
|
|
1195
1217
|
assert dataset.extras["dcat"].get("rights") is None
|
|
1196
1218
|
for resource in dataset.resources:
|
|
1197
1219
|
assert resource.extras["dcat"].get("rights") is None
|
|
1220
|
+
|
|
1221
|
+
# Additional INSPIRE tag due to the dataset having a GEMET INSPIRE theme
|
|
1222
|
+
assert "inspire" in dataset.tags
|
udata/rdf.py
CHANGED
|
@@ -128,6 +128,12 @@ EU_HVD_CATEGORIES = {
|
|
|
128
128
|
HVD_LEGISLATION = "http://data.europa.eu/eli/reg_impl/2023/138/oj"
|
|
129
129
|
TAG_TO_EU_HVD_CATEGORIES = {slugify_tag(EU_HVD_CATEGORIES[uri]): uri for uri in EU_HVD_CATEGORIES}
|
|
130
130
|
|
|
131
|
+
INSPIRE_GEMET_THEME_NAMESPACE = "http://inspire.ec.europa.eu/theme"
|
|
132
|
+
INSPIRE_GEMET_SCHEME_URIS = [
|
|
133
|
+
INSPIRE_GEMET_THEME_NAMESPACE,
|
|
134
|
+
"http://www.eionet.europa.eu/gemet/inspire_themes",
|
|
135
|
+
]
|
|
136
|
+
|
|
131
137
|
AGENT_ROLE_TO_RDF_PREDICATE = {
|
|
132
138
|
"contact": DCAT.contactPoint,
|
|
133
139
|
"publisher": DCT.publisher,
|
|
@@ -303,16 +309,37 @@ def theme_labels_from_rdf(rdf):
|
|
|
303
309
|
"""
|
|
304
310
|
Get theme labels to use as keywords.
|
|
305
311
|
Map HVD keywords from known URIs resources if HVD support is activated.
|
|
312
|
+
Map INSPIRE keyword from known themes if INSPIRE support is activated.
|
|
313
|
+
- An INSPIRE dataset is a dataset with a theme INSPIRE encoded with gmd:descriptiveKeywords/gmd:MD_Keywords.
|
|
314
|
+
In DCAT, it is shown as a DCAT.theme with a SKOS.inScheme pointing to to the INSPIRE thesaurus.
|
|
315
|
+
We filter on this thesaurus based on its name (expecting "GEMET - INSPIRE themes, version 1.0") or its uri.
|
|
306
316
|
"""
|
|
307
317
|
for theme in rdf.objects(DCAT.theme):
|
|
308
318
|
if isinstance(theme, RdfResource):
|
|
319
|
+
label = rdf_value(theme, SKOS.prefLabel)
|
|
309
320
|
uri = theme.identifier.toPython()
|
|
310
321
|
if current_app.config["HVD_SUPPORT"] and uri in EU_HVD_CATEGORIES:
|
|
322
|
+
# Map label from EU HVD categories
|
|
311
323
|
label = EU_HVD_CATEGORIES[uri]
|
|
312
324
|
# Additionnally yield hvd keyword
|
|
313
325
|
yield "hvd"
|
|
314
|
-
|
|
315
|
-
|
|
326
|
+
if current_app.config["INSPIRE_SUPPORT"]:
|
|
327
|
+
if uri.startswith(INSPIRE_GEMET_THEME_NAMESPACE):
|
|
328
|
+
yield "inspire"
|
|
329
|
+
else:
|
|
330
|
+
# Check if the theme belongs to the GEMET INSPIRE scheme
|
|
331
|
+
if scheme := theme.value(SKOS.inScheme):
|
|
332
|
+
scheme_title = (
|
|
333
|
+
rdf_value(scheme, DCT.title)
|
|
334
|
+
or rdf_value(scheme, SKOS.prefLabel)
|
|
335
|
+
or rdf_value(scheme, RDFS.label)
|
|
336
|
+
)
|
|
337
|
+
scheme_uri = scheme.identifier.toPython()
|
|
338
|
+
if (
|
|
339
|
+
scheme_title
|
|
340
|
+
and scheme_title.lower() == "gemet - inspire themes, version 1.0"
|
|
341
|
+
) or scheme_uri in INSPIRE_GEMET_SCHEME_URIS:
|
|
342
|
+
yield "inspire"
|
|
316
343
|
else:
|
|
317
344
|
label = theme.toPython()
|
|
318
345
|
if label:
|
udata/settings.py
CHANGED
|
@@ -288,6 +288,8 @@ class Defaults(object):
|
|
|
288
288
|
|
|
289
289
|
DELAY_BEFORE_REMINDER_NOTIFICATION = 30 # Days
|
|
290
290
|
|
|
291
|
+
DELAY_BEFORE_APPEARING_IN_RSS_FEED = 10 # Hours
|
|
292
|
+
|
|
291
293
|
# Harvest settings
|
|
292
294
|
###########################################################################
|
|
293
295
|
HARVEST_ENABLE_MANUAL_RUN = False
|
|
@@ -324,9 +326,14 @@ class Defaults(object):
|
|
|
324
326
|
S3_ACCESS_KEY_ID = None
|
|
325
327
|
S3_SECRET_ACCESS_KEY = None
|
|
326
328
|
|
|
327
|
-
# Specific support for hvd
|
|
329
|
+
# Specific support for hvd:
|
|
330
|
+
# - map HVD categories URIs to keywords
|
|
328
331
|
HVD_SUPPORT = True
|
|
329
332
|
|
|
333
|
+
# Specific support for inspire:
|
|
334
|
+
# - add inspire keyword during harvest if GEMETE INSPIRE thesaurus is used in DCAT.theme
|
|
335
|
+
INSPIRE_SUPPORT = True
|
|
336
|
+
|
|
330
337
|
ACTIVATE_TERRITORIES = False
|
|
331
338
|
# The order is important to compute parents/children, smaller first.
|
|
332
339
|
HANDLED_LEVELS = tuple()
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
from datetime import datetime, timedelta
|
|
2
2
|
|
|
3
3
|
import feedparser
|
|
4
|
+
import pytest
|
|
4
5
|
from flask import url_for
|
|
5
6
|
from werkzeug.test import TestResponse
|
|
6
7
|
|
|
@@ -663,7 +664,9 @@ class DataserviceAPITest(APITestCase):
|
|
|
663
664
|
|
|
664
665
|
|
|
665
666
|
class DataservicesFeedAPItest(APITestCase):
|
|
667
|
+
@pytest.mark.options(DELAY_BEFORE_APPEARING_IN_RSS_FEED=10)
|
|
666
668
|
def test_recent_feed(self):
|
|
669
|
+
# We have a 10 hours delay for a new object to appear in feed. A newly created one shouldn't appear.
|
|
667
670
|
DataserviceFactory(title="A", created_at=datetime.utcnow())
|
|
668
671
|
DataserviceFactory(title="B", created_at=datetime.utcnow() - timedelta(days=2))
|
|
669
672
|
DataserviceFactory(title="C", created_at=datetime.utcnow() - timedelta(days=1))
|
|
@@ -673,11 +676,11 @@ class DataservicesFeedAPItest(APITestCase):
|
|
|
673
676
|
|
|
674
677
|
feed = feedparser.parse(response.data)
|
|
675
678
|
|
|
676
|
-
self.assertEqual(len(feed.entries),
|
|
677
|
-
self.assertEqual(feed.entries[0].title, "
|
|
678
|
-
self.assertEqual(feed.entries[1].title, "
|
|
679
|
-
self.assertEqual(feed.entries[2].title, "B")
|
|
679
|
+
self.assertEqual(len(feed.entries), 2)
|
|
680
|
+
self.assertEqual(feed.entries[0].title, "C")
|
|
681
|
+
self.assertEqual(feed.entries[1].title, "B")
|
|
680
682
|
|
|
683
|
+
@pytest.mark.options(DELAY_BEFORE_APPEARING_IN_RSS_FEED=0)
|
|
681
684
|
def test_recent_feed_owner(self):
|
|
682
685
|
owner = UserFactory()
|
|
683
686
|
DataserviceFactory(owner=owner)
|
|
@@ -695,6 +698,7 @@ class DataservicesFeedAPItest(APITestCase):
|
|
|
695
698
|
self.assertEqual(author.name, owner.fullname)
|
|
696
699
|
self.assertEqual(author.href, owner.url_for())
|
|
697
700
|
|
|
701
|
+
@pytest.mark.options(DELAY_BEFORE_APPEARING_IN_RSS_FEED=0)
|
|
698
702
|
def test_recent_feed_org(self):
|
|
699
703
|
owner = UserFactory()
|
|
700
704
|
org = OrganizationFactory()
|
|
@@ -36,6 +36,7 @@ from udata.core.dataset.models import (
|
|
|
36
36
|
ResourceMixin,
|
|
37
37
|
)
|
|
38
38
|
from udata.core.organization.factories import OrganizationFactory
|
|
39
|
+
from udata.core.organization.models import OrganizationBadge
|
|
39
40
|
from udata.core.spatial.factories import GeoLevelFactory, SpatialCoverageFactory
|
|
40
41
|
from udata.core.topic.factories import TopicElementDatasetFactory, TopicFactory
|
|
41
42
|
from udata.core.user.factories import AdminFactory, UserFactory
|
|
@@ -1322,31 +1323,47 @@ class DatasetAPITest(APITestCase):
|
|
|
1322
1323
|
|
|
1323
1324
|
|
|
1324
1325
|
class DatasetsFeedAPItest(APITestCase):
|
|
1326
|
+
@pytest.mark.options(DELAY_BEFORE_APPEARING_IN_RSS_FEED=10)
|
|
1325
1327
|
def test_recent_feed(self):
|
|
1328
|
+
certified_org = OrganizationFactory(badges=[OrganizationBadge(kind="certified")])
|
|
1329
|
+
# We have a 10 hours delay for a new object to appear in feed. A newly created one shouldn't appear.
|
|
1326
1330
|
DatasetFactory(
|
|
1327
1331
|
title="A", resources=[ResourceFactory()], created_at_internal=datetime.utcnow()
|
|
1328
1332
|
)
|
|
1333
|
+
# Except in the case of a new dataset published by a certified organization
|
|
1329
1334
|
DatasetFactory(
|
|
1330
1335
|
title="B",
|
|
1331
|
-
|
|
1332
|
-
|
|
1336
|
+
created_at_internal=datetime.utcnow(),
|
|
1337
|
+
organization=certified_org,
|
|
1333
1338
|
)
|
|
1334
1339
|
DatasetFactory(
|
|
1335
1340
|
title="C",
|
|
1336
|
-
|
|
1341
|
+
created_at_internal=datetime.utcnow() - timedelta(days=2),
|
|
1342
|
+
)
|
|
1343
|
+
DatasetFactory(
|
|
1344
|
+
title="D",
|
|
1337
1345
|
created_at_internal=datetime.utcnow() - timedelta(days=1),
|
|
1338
1346
|
)
|
|
1347
|
+
# Even if dataset E is created more recently than D, it should appear after in the feed, since it doesn't have a delay
|
|
1348
|
+
# before appearing in the field because it is published by a certified organization
|
|
1349
|
+
DatasetFactory(
|
|
1350
|
+
title="E",
|
|
1351
|
+
created_at_internal=datetime.utcnow() - timedelta(hours=23),
|
|
1352
|
+
organization=certified_org,
|
|
1353
|
+
)
|
|
1339
1354
|
|
|
1340
1355
|
response = self.get(url_for("api.recent_datasets_atom_feed"))
|
|
1341
1356
|
self.assert200(response)
|
|
1342
1357
|
|
|
1343
1358
|
feed = feedparser.parse(response.data)
|
|
1344
1359
|
|
|
1345
|
-
self.assertEqual(len(feed.entries),
|
|
1346
|
-
self.assertEqual(feed.entries[0].title, "
|
|
1347
|
-
self.assertEqual(feed.entries[1].title, "
|
|
1348
|
-
self.assertEqual(feed.entries[2].title, "
|
|
1360
|
+
self.assertEqual(len(feed.entries), 4)
|
|
1361
|
+
self.assertEqual(feed.entries[0].title, "B")
|
|
1362
|
+
self.assertEqual(feed.entries[1].title, "D")
|
|
1363
|
+
self.assertEqual(feed.entries[2].title, "E")
|
|
1364
|
+
self.assertEqual(feed.entries[3].title, "C")
|
|
1349
1365
|
|
|
1366
|
+
@pytest.mark.options(DELAY_BEFORE_APPEARING_IN_RSS_FEED=0)
|
|
1350
1367
|
def test_recent_feed_owner(self):
|
|
1351
1368
|
owner = UserFactory()
|
|
1352
1369
|
DatasetFactory(owner=owner, resources=[ResourceFactory()])
|
|
@@ -1364,6 +1381,7 @@ class DatasetsFeedAPItest(APITestCase):
|
|
|
1364
1381
|
self.assertEqual(author.name, owner.fullname)
|
|
1365
1382
|
self.assertEqual(author.href, owner.url_for())
|
|
1366
1383
|
|
|
1384
|
+
@pytest.mark.options(DELAY_BEFORE_APPEARING_IN_RSS_FEED=0)
|
|
1367
1385
|
def test_recent_feed_org(self):
|
|
1368
1386
|
owner = UserFactory()
|
|
1369
1387
|
org = OrganizationFactory()
|
|
@@ -572,7 +572,9 @@ class ReuseAPITest:
|
|
|
572
572
|
|
|
573
573
|
|
|
574
574
|
class ReusesFeedAPItest(APITestCase):
|
|
575
|
+
@pytest.mark.options(DELAY_BEFORE_APPEARING_IN_RSS_FEED=10)
|
|
575
576
|
def test_recent_feed(self):
|
|
577
|
+
# We have a 10 hours delay for a new object to appear in feed. A newly created one shouldn't appear.
|
|
576
578
|
ReuseFactory(title="A", datasets=[DatasetFactory()], created_at=datetime.utcnow())
|
|
577
579
|
ReuseFactory(
|
|
578
580
|
title="B", datasets=[DatasetFactory()], created_at=datetime.utcnow() - timedelta(days=2)
|
|
@@ -586,11 +588,11 @@ class ReusesFeedAPItest(APITestCase):
|
|
|
586
588
|
|
|
587
589
|
feed = feedparser.parse(response.data)
|
|
588
590
|
|
|
589
|
-
self.assertEqual(len(feed.entries),
|
|
590
|
-
self.assertEqual(feed.entries[0].title, "
|
|
591
|
-
self.assertEqual(feed.entries[1].title, "
|
|
592
|
-
self.assertEqual(feed.entries[2].title, "B")
|
|
591
|
+
self.assertEqual(len(feed.entries), 2)
|
|
592
|
+
self.assertEqual(feed.entries[0].title, "C")
|
|
593
|
+
self.assertEqual(feed.entries[1].title, "B")
|
|
593
594
|
|
|
595
|
+
@pytest.mark.options(DELAY_BEFORE_APPEARING_IN_RSS_FEED=0)
|
|
594
596
|
def test_recent_feed_owner(self):
|
|
595
597
|
owner = UserFactory()
|
|
596
598
|
ReuseFactory(owner=owner, datasets=[DatasetFactory()])
|
|
@@ -608,6 +610,7 @@ class ReusesFeedAPItest(APITestCase):
|
|
|
608
610
|
self.assertEqual(author.name, owner.fullname)
|
|
609
611
|
self.assertEqual(author.href, owner.url_for())
|
|
610
612
|
|
|
613
|
+
@pytest.mark.options(DELAY_BEFORE_APPEARING_IN_RSS_FEED=0)
|
|
611
614
|
def test_recent_feed_org(self):
|
|
612
615
|
owner = UserFactory()
|
|
613
616
|
org = OrganizationFactory()
|
udata/utils.py
CHANGED
|
@@ -3,7 +3,7 @@ import itertools
|
|
|
3
3
|
import math
|
|
4
4
|
import re
|
|
5
5
|
from collections import Counter
|
|
6
|
-
from datetime import date, datetime
|
|
6
|
+
from datetime import date, datetime, timedelta
|
|
7
7
|
from math import ceil
|
|
8
8
|
from typing import Any, Hashable
|
|
9
9
|
from uuid import UUID, uuid4
|
|
@@ -18,7 +18,8 @@ from faker import Faker
|
|
|
18
18
|
from faker.config import PROVIDERS
|
|
19
19
|
from faker.providers import BaseProvider
|
|
20
20
|
from faker.providers.lorem.la import Provider as LoremProvider
|
|
21
|
-
from flask import abort, request
|
|
21
|
+
from flask import abort, current_app, request
|
|
22
|
+
from mongoengine.fields import BaseQuerySet
|
|
22
23
|
|
|
23
24
|
from udata import tags
|
|
24
25
|
|
|
@@ -382,6 +383,55 @@ def id_or_404(object_id):
|
|
|
382
383
|
abort(404)
|
|
383
384
|
|
|
384
385
|
|
|
386
|
+
def get_rss_feed_list(queryset: BaseQuerySet, created_at_field: str) -> list[Any]:
|
|
387
|
+
"""
|
|
388
|
+
Return a list of recent elements for a RSS field.
|
|
389
|
+
|
|
390
|
+
We add a delay before a new element appears in feed in order to allow for post-publication moderation.
|
|
391
|
+
The delay is not taken into account if the element is published by a certified organization.
|
|
392
|
+
"""
|
|
393
|
+
from udata.core.organization.constants import CERTIFIED
|
|
394
|
+
from udata.core.site.models import current_site
|
|
395
|
+
from udata.models import Organization
|
|
396
|
+
|
|
397
|
+
certifed_orgs = Organization.objects(badges__kind=CERTIFIED).only("id")
|
|
398
|
+
|
|
399
|
+
created_delay = datetime.utcnow() - timedelta(
|
|
400
|
+
hours=current_app.config["DELAY_BEFORE_APPEARING_IN_RSS_FEED"]
|
|
401
|
+
)
|
|
402
|
+
elements_with_delay = list(
|
|
403
|
+
queryset.filter(
|
|
404
|
+
**{
|
|
405
|
+
f"{created_at_field}__lte": created_delay,
|
|
406
|
+
"organization__nin": certifed_orgs,
|
|
407
|
+
}
|
|
408
|
+
)
|
|
409
|
+
.order_by(f"-{created_at_field}")
|
|
410
|
+
.limit(current_site.feed_size)
|
|
411
|
+
)
|
|
412
|
+
elements_without_delay = list(
|
|
413
|
+
queryset.filter(organization__in=certifed_orgs)
|
|
414
|
+
.order_by(f"-{created_at_field}")
|
|
415
|
+
.limit(current_site.feed_size)
|
|
416
|
+
)
|
|
417
|
+
|
|
418
|
+
# We need to merge the two lists manually, compensating for the delay, else elements with delay may not show
|
|
419
|
+
# if new elements have been published by certified organization in the meantime
|
|
420
|
+
def get_sort_key(element):
|
|
421
|
+
has_delay = not element.organization or not element.organization.certified
|
|
422
|
+
if has_delay:
|
|
423
|
+
return getattr(element, created_at_field) + timedelta(
|
|
424
|
+
hours=current_app.config["DELAY_BEFORE_APPEARING_IN_RSS_FEED"]
|
|
425
|
+
)
|
|
426
|
+
return getattr(element, created_at_field)
|
|
427
|
+
|
|
428
|
+
elements = sorted(
|
|
429
|
+
[*elements_with_delay, *elements_without_delay], reverse=True, key=get_sort_key
|
|
430
|
+
)[: current_site.feed_size]
|
|
431
|
+
|
|
432
|
+
return elements
|
|
433
|
+
|
|
434
|
+
|
|
385
435
|
def wants_json() -> bool:
|
|
386
436
|
if request.is_json:
|
|
387
437
|
return True
|
|
@@ -7,17 +7,17 @@ udata/errors.py,sha256=E8W7b4PH7c5B85g_nsUMt8fHqMVpDFOZFkO6wMPl6bA,117
|
|
|
7
7
|
udata/factories.py,sha256=MoklZnU8iwNL25dm3JsoXhoQs1PQWSVYL1WvcUBtJqM,492
|
|
8
8
|
udata/i18n.py,sha256=bC9ajf66YgcYoJffvresLZLa32rb6NsY-JGMtFiVsG4,8163
|
|
9
9
|
udata/mail.py,sha256=Huhx_1QthJkLvuRUuP6jqb5Qq5R4iSmqeEpLVO9ZkQ4,2671
|
|
10
|
-
udata/rdf.py,sha256=
|
|
10
|
+
udata/rdf.py,sha256=dQSYVS2NrMRE6fsrbGwXl_roM0k6jtt1JPtMhHxeVGg,20970
|
|
11
11
|
udata/routing.py,sha256=Hnc1ktmKVS-RUHNKw2zYTft2HJ903FhjtlcenQ9igwI,8044
|
|
12
12
|
udata/sentry.py,sha256=ekcxqUSqxfM98TtvCsPaOoX5i2l6PEcYt7kb4l3od-Q,3223
|
|
13
|
-
udata/settings.py,sha256=
|
|
13
|
+
udata/settings.py,sha256=A6CRPN99_EMK7bXTf4bdZI7NZEuAHECUkxChz7FWUkw,21904
|
|
14
14
|
udata/sitemap.py,sha256=oRRWoPI7ZsFFnUAOqGT1YuXFFKHBe8EcRnUCNHD7xjM,979
|
|
15
15
|
udata/tags.py,sha256=ydq4uokd6bzdeGVSpEXASVtGvDfO2LfQs9mptvvKJCM,631
|
|
16
16
|
udata/tasks.py,sha256=Sv01dhvATtq_oHOBp3J1j1VT1HQe0Pab7zxwIeIdKoo,5122
|
|
17
17
|
udata/terms.md,sha256=nFx978tUQ3vTEv6POykXaZvcQ5e_gcvmO4ZgcfbSWXo,187
|
|
18
18
|
udata/tracking.py,sha256=WOcqA1RlHN8EPFuEc2kNau54mec4-pvi-wUFrMXevzg,345
|
|
19
19
|
udata/uris.py,sha256=u05ltyLv4ZolaNMbq58kcLizMXWC10mRpJFYzChUHJU,4440
|
|
20
|
-
udata/utils.py,sha256=
|
|
20
|
+
udata/utils.py,sha256=43-q1yZ8Phd6VTCNn7a4sIx1qs_oqVvIFg-RQb0qvgQ,13243
|
|
21
21
|
udata/worker.py,sha256=K-Wafye5-uXP4kQlffRKws2J9YbJ6m6n2QjcVsY8Nsg,118
|
|
22
22
|
udata/wsgi.py,sha256=MY8en9K9eDluvJYUxTdzqSDoYaDgCVZ69ZcUvxAvgqA,77
|
|
23
23
|
udata/api/__init__.py,sha256=Rs1V6KqWUcZliZnoIrVKtq15Bfb8ZaOVOFuGhcv0SK8,11948
|
|
@@ -79,7 +79,7 @@ udata/core/contact_point/forms.py,sha256=oBe1agSJFyx2QRgYzPRg2A7qVscaBTaKG4V-AyI
|
|
|
79
79
|
udata/core/contact_point/models.py,sha256=Xqmqg7S13gcaKxiQT52WHeQEHTaUDDGIXInXyqNh4Po,854
|
|
80
80
|
udata/core/dataservices/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
81
81
|
udata/core/dataservices/activities.py,sha256=wcYQCyYpKciCz99VqQqKti72a5Fyhc-AvDZBWdF0KUc,1763
|
|
82
|
-
udata/core/dataservices/api.py,sha256=
|
|
82
|
+
udata/core/dataservices/api.py,sha256=FhPF-oq0rgLo_trkttrz5TsFSLLNVGX5KH5Jzid1sDA,9988
|
|
83
83
|
udata/core/dataservices/apiv2.py,sha256=pd4UWF6m7pQ5nmBRFmeiDyedhZSngz211to6qTscVsA,1205
|
|
84
84
|
udata/core/dataservices/constants.py,sha256=SxetyqvbWQlb-T9Bqn7mdfzV3vS-is7s56rzmGzTXY0,1029
|
|
85
85
|
udata/core/dataservices/csv.py,sha256=HWI2JrN_Vuw0te9FHlJ6eyqcRcKHOKXuzg45D4Ti6F0,1106
|
|
@@ -92,7 +92,7 @@ udata/core/dataservices/tasks.py,sha256=fHG1r5ymfJRXJ_Lug6je3VKZoK30XKXE2rQ8x0R-
|
|
|
92
92
|
udata/core/dataset/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
93
93
|
udata/core/dataset/actions.py,sha256=mX6xox0PiMrbcAPZ3VZsI26rfM-ciYfEXxN6sqqImKA,1222
|
|
94
94
|
udata/core/dataset/activities.py,sha256=eGxMUnC47YHxTgcls6igQ3qP7cYgwFtPfj0asCylGsI,3315
|
|
95
|
-
udata/core/dataset/api.py,sha256=
|
|
95
|
+
udata/core/dataset/api.py,sha256=o8Q7zrtR3ESbGO9aMa8FPGw3qBHujvV8Kt38axv5rsc,35534
|
|
96
96
|
udata/core/dataset/api_fields.py,sha256=p7ZnmGNImZ4sgZTpoyHpI0CgOukpEIx8QdGnxlmgl2I,18032
|
|
97
97
|
udata/core/dataset/apiv2.py,sha256=1H4557ZMi6rwEyrwB1Ha20m0bf3Avhg_vDLiDQt5Fi0,21030
|
|
98
98
|
udata/core/dataset/commands.py,sha256=3mKSdJ-M7ggdG29AVn77C4ouZanbYoqkTaGQoBKOp3s,3471
|
|
@@ -181,7 +181,7 @@ udata/core/reports/constants.py,sha256=LRZSX3unyqZeB4yQjK3ws_hGbJcXYk4bu1Rhnhi5D
|
|
|
181
181
|
udata/core/reports/models.py,sha256=xT6zSXJx8vG0c8cu4VWUQvENUwajusFlPQmenslR3lQ,2662
|
|
182
182
|
udata/core/reuse/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
183
183
|
udata/core/reuse/activities.py,sha256=5D7cV-hGZnzHsp8hohZqqgK3RSGQpfAqJ_Wfq_AYfM8,1420
|
|
184
|
-
udata/core/reuse/api.py,sha256=
|
|
184
|
+
udata/core/reuse/api.py,sha256=myUQiuXffXAZK6tAL48N-kuri2uK0aiqlyiSTSp7N-w,12447
|
|
185
185
|
udata/core/reuse/api_fields.py,sha256=ccym6v9Ap68PlHZmIMMtHQFnEyV7Gbxrfdw0b6rj51A,1232
|
|
186
186
|
udata/core/reuse/apiv2.py,sha256=mgvL1ZBts872zNSo0Wh3AkNAh2ldzP9yQkRY3WfzGPM,944
|
|
187
187
|
udata/core/reuse/constants.py,sha256=JgDBrjOKSt9q0auv9rjzbGsch83H-Oi8YXAKeI5hO4o,1215
|
|
@@ -324,7 +324,7 @@ udata/harvest/tests/person.jsonld,sha256=I7Ynh-PQlNeD51I1LrCgYOEjhL-WBeb65xzIE_s
|
|
|
324
324
|
udata/harvest/tests/test_actions.py,sha256=d5TTFTbs4PdBydWICqDtfoeo3zLyzcNDzv4aMH9spxo,25881
|
|
325
325
|
udata/harvest/tests/test_api.py,sha256=gSuICkPy3KVRUhHAyudXVf_gLwiB7SoriUp3DLXWDdA,21611
|
|
326
326
|
udata/harvest/tests/test_base_backend.py,sha256=ow8ecGtD836mUqyPWYjkS5nx0STyT5RMLgBdDyOhts4,19233
|
|
327
|
-
udata/harvest/tests/test_dcat_backend.py,sha256=
|
|
327
|
+
udata/harvest/tests/test_dcat_backend.py,sha256=WB7jSjRuFvSAjskKVhW5lUfiJHomL3hwebSJ3f8ubEI,51842
|
|
328
328
|
udata/harvest/tests/test_filters.py,sha256=PT2qopEIoXsqi8MsNDRuhNH7jGXiQo8r0uJrCOUd4aM,2465
|
|
329
329
|
udata/harvest/tests/test_models.py,sha256=f9NRR2_S4oZFgF8qOumg0vv-lpnEBJbI5vNtcwFdSqM,831
|
|
330
330
|
udata/harvest/tests/test_notifications.py,sha256=MMzTzkv-GXMNFeOwAi31rdTsAXyLCLOSna41zOtaJG0,816
|
|
@@ -344,7 +344,7 @@ udata/harvest/tests/csw_dcat/geonetworkv4-page-1.xml,sha256=k2pKidlQvJpoltGFm9HN
|
|
|
344
344
|
udata/harvest/tests/csw_dcat/geonetworkv4-page-3.xml,sha256=fsN0E4TVd_ts-sYA612yBP-gRAwpyQWqJdNm7ohczbs,20945
|
|
345
345
|
udata/harvest/tests/csw_dcat/geonetworkv4-page-5.xml,sha256=0VmPp1kspik7YAmOFyr-3yJLzWGA6kuQp_x_w-W385o,21213
|
|
346
346
|
udata/harvest/tests/dcat/bnodes.jsonld,sha256=Leqny-ccp30564yojQYYckw_HKbhR0f5qUCaavc2ruE,7964
|
|
347
|
-
udata/harvest/tests/dcat/bnodes.xml,sha256=
|
|
347
|
+
udata/harvest/tests/dcat/bnodes.xml,sha256=k1AGGPD8P4qSIxzS2vorJMNnf1sw4Deq3Kwm_rDJMaM,12137
|
|
348
348
|
udata/harvest/tests/dcat/catalog.xml,sha256=7bXxQDAu-BRppqIq4WAu6QAdOGOSbf900zBzJttP30k,12785
|
|
349
349
|
udata/harvest/tests/dcat/evian.json,sha256=R3RxP5azUuf9aZ9fU7n6iJkfbJ6oj-Zej2cjOtkYr8M,16647
|
|
350
350
|
udata/harvest/tests/dcat/flat.jsonld,sha256=BAw08MDhtW9Px3q6RAoTIqO_OwJmAwBS9EpC8BY_x98,8459
|
|
@@ -516,14 +516,14 @@ udata/tests/api/test_activities_api.py,sha256=GzyznB1rHrWfPH-te5RKmA3Ezlo5jlP3z7
|
|
|
516
516
|
udata/tests/api/test_auth_api.py,sha256=Mue5EneRp3MbD-ahcXI3YaRNYPcxzHMw-hYpY0RtGVU,25569
|
|
517
517
|
udata/tests/api/test_base_api.py,sha256=2w_vz0eEuq3P3aN-ByvxGc3VZAo7XtgatFfcrzf2uEU,2244
|
|
518
518
|
udata/tests/api/test_contact_points.py,sha256=Sbb486RTN7HVycna_XB60OnURPSNc7xUity26XsYA4k,8766
|
|
519
|
-
udata/tests/api/test_dataservices_api.py,sha256=
|
|
520
|
-
udata/tests/api/test_datasets_api.py,sha256=
|
|
519
|
+
udata/tests/api/test_dataservices_api.py,sha256=pB7Kpf0qLhsX4z5hYj3BykYKauEMpESWmrlLlxTOmWw,29221
|
|
520
|
+
udata/tests/api/test_datasets_api.py,sha256=Qlnitk2WmLDJJ7m7ULfV0UX8ml9VXev8Bd1PMv7v6_A,105949
|
|
521
521
|
udata/tests/api/test_fields.py,sha256=OW85Z5MES5HeWOpapeem8OvR1cIcrqW-xMWpdZO4LZ8,1033
|
|
522
522
|
udata/tests/api/test_follow_api.py,sha256=4nFXG5pZ_Hf2PJ4KEdHJX_uggjc9RpB8v0fidkAcw9I,5792
|
|
523
523
|
udata/tests/api/test_me_api.py,sha256=YPd8zmR3zwJKtpSqz8nY1nOOMyXs66INeBwyhg5D0Us,13846
|
|
524
524
|
udata/tests/api/test_organizations_api.py,sha256=tI6OiLlEZ0ydn1TZxcb6k3__PQwomfX8nBHfAj7Vf90,44126
|
|
525
525
|
udata/tests/api/test_reports_api.py,sha256=fCSz9NwMXBs6cxdXBVVI6y564AtovmZYw3xkgxQ9KE8,6217
|
|
526
|
-
udata/tests/api/test_reuses_api.py,sha256=
|
|
526
|
+
udata/tests/api/test_reuses_api.py,sha256=B7_LA8Cuq_MqkE3C_hdvmugFXwzPJ9yKDckaZGpFQvU,27471
|
|
527
527
|
udata/tests/api/test_swagger.py,sha256=eE6La9qdTYTIUFevRVPJgtj17Jq_8uOlsDwzCNR0LL8,760
|
|
528
528
|
udata/tests/api/test_tags_api.py,sha256=36zEBgthVEn6pctJ0kDgPmEaUr-iqRAHeZRcRG2LEXQ,2425
|
|
529
529
|
udata/tests/api/test_transfer_api.py,sha256=-OLv-KjyLZL14J8UHl-ak_sYUj6wFiZWyoXC2SMXmEQ,7503
|
|
@@ -625,9 +625,9 @@ udata/translations/pt/LC_MESSAGES/udata.mo,sha256=nv80xZLfIfUsSOMBcr29L268FDc_Gt
|
|
|
625
625
|
udata/translations/pt/LC_MESSAGES/udata.po,sha256=bUp-7Ray8t8ALgJk3Icw1jmiGIc9_pEJQHiGw_2EU2o,50989
|
|
626
626
|
udata/translations/sr/LC_MESSAGES/udata.mo,sha256=Y_XpUxD074wXc63oJTnoVOyOQ2lmBxl-MrgluZ0Qdw4,27961
|
|
627
627
|
udata/translations/sr/LC_MESSAGES/udata.po,sha256=qh8mrz9AFuVQtXYSSP4QWsXLM_Lv3EHVifHT1NflWXY,57529
|
|
628
|
-
udata-12.0.2.
|
|
629
|
-
udata-12.0.2.
|
|
630
|
-
udata-12.0.2.
|
|
631
|
-
udata-12.0.2.
|
|
632
|
-
udata-12.0.2.
|
|
633
|
-
udata-12.0.2.
|
|
628
|
+
udata-12.0.2.dev15.dist-info/licenses/LICENSE,sha256=V8j_M8nAz8PvAOZQocyRDX7keai8UJ9skgmnwqETmdY,34520
|
|
629
|
+
udata-12.0.2.dev15.dist-info/METADATA,sha256=FbfUH9JcbjqFCQjt6f5RbgVob95aMfPdRE7L6jSSCTU,5174
|
|
630
|
+
udata-12.0.2.dev15.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
631
|
+
udata-12.0.2.dev15.dist-info/entry_points.txt,sha256=v2u12qO11i2lyLNIp136WmLJ-NHT-Kew3Duu8J-AXPM,614
|
|
632
|
+
udata-12.0.2.dev15.dist-info/top_level.txt,sha256=EF6CE6YSHd_og-8LCEA4q25ALUpWVe8D0okOLdMAE3A,6
|
|
633
|
+
udata-12.0.2.dev15.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|