udata 11.1.2.dev8__py3-none-any.whl → 11.1.2.dev11__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of udata might be problematic. Click here for more details.
- udata/api/oauth2.py +22 -3
- udata/app.py +3 -0
- udata/auth/__init__.py +11 -0
- udata/auth/forms.py +70 -3
- udata/auth/mails.py +6 -0
- udata/auth/proconnect.py +127 -0
- udata/auth/views.py +57 -2
- udata/core/__init__.py +2 -0
- udata/core/captchetat.py +80 -0
- udata/core/dataset/api.py +2 -2
- udata/core/dataset/api_fields.py +3 -4
- udata/core/dataset/apiv2.py +6 -6
- udata/core/dataset/commands.py +0 -10
- udata/core/dataset/constants.py +124 -38
- udata/core/dataset/factories.py +2 -1
- udata/core/dataset/forms.py +14 -10
- udata/core/dataset/models.py +8 -36
- udata/core/dataset/rdf.py +76 -54
- udata/core/dataset/tasks.py +2 -50
- udata/cors.py +19 -2
- udata/harvest/backends/ckan/harvesters.py +10 -14
- udata/harvest/backends/maaf.py +15 -14
- udata/harvest/tests/ckan/test_ckan_backend.py +4 -3
- udata/harvest/tests/test_dcat_backend.py +3 -2
- udata/i18n.py +7 -32
- udata/migrations/2025-09-04-update-legacy-frequencies.py +36 -0
- udata/settings.py +27 -0
- udata/templates/security/email/reset_instructions.html +1 -1
- udata/templates/security/email/reset_instructions.txt +1 -1
- udata/tests/api/test_datasets_api.py +41 -12
- udata/tests/dataset/test_dataset_model.py +17 -53
- udata/tests/dataset/test_dataset_rdf.py +27 -28
- udata/translations/udata.pot +226 -150
- udata/utils.py +8 -1
- {udata-11.1.2.dev8.dist-info → udata-11.1.2.dev11.dist-info}/METADATA +1 -1
- {udata-11.1.2.dev8.dist-info → udata-11.1.2.dev11.dist-info}/RECORD +40 -40
- udata/templates/mail/frequency_reminder.html +0 -34
- udata/templates/mail/frequency_reminder.txt +0 -18
- udata/tests/test_i18n.py +0 -93
- {udata-11.1.2.dev8.dist-info → udata-11.1.2.dev11.dist-info}/WHEEL +0 -0
- {udata-11.1.2.dev8.dist-info → udata-11.1.2.dev11.dist-info}/entry_points.txt +0 -0
- {udata-11.1.2.dev8.dist-info → udata-11.1.2.dev11.dist-info}/licenses/LICENSE +0 -0
- {udata-11.1.2.dev8.dist-info → udata-11.1.2.dev11.dist-info}/top_level.txt +0 -0
|
@@ -4,19 +4,14 @@ from urllib.parse import urljoin
|
|
|
4
4
|
from uuid import UUID
|
|
5
5
|
|
|
6
6
|
from udata import uris
|
|
7
|
-
from udata.
|
|
8
|
-
from udata.i18n import lazy_gettext as _
|
|
9
|
-
|
|
10
|
-
try:
|
|
11
|
-
from udata.core.dataset.constants import UPDATE_FREQUENCIES
|
|
12
|
-
except ImportError:
|
|
13
|
-
# legacy import of constants in udata
|
|
14
|
-
from udata.models import UPDATE_FREQUENCIES
|
|
7
|
+
from udata.core.dataset.constants import UpdateFrequency
|
|
15
8
|
from udata.core.dataset.models import HarvestDatasetMetadata, HarvestResourceMetadata
|
|
16
9
|
from udata.core.dataset.rdf import frequency_from_rdf
|
|
17
10
|
from udata.frontend.markdown import parse_html
|
|
18
11
|
from udata.harvest.backends.base import BaseBackend, HarvestFilter
|
|
19
12
|
from udata.harvest.exceptions import HarvestException, HarvestSkipException
|
|
13
|
+
from udata.harvest.models import HarvestItem
|
|
14
|
+
from udata.i18n import lazy_gettext as _
|
|
20
15
|
from udata.models import GeoZone, License, Resource, SpatialCoverage, db
|
|
21
16
|
from udata.utils import daterange_end, daterange_start, get_by
|
|
22
17
|
|
|
@@ -193,14 +188,15 @@ class CkanBackend(BaseBackend):
|
|
|
193
188
|
log.debug("spatial-uri value not handled: %s", value)
|
|
194
189
|
elif key == "frequency":
|
|
195
190
|
# Update frequency
|
|
196
|
-
freq
|
|
197
|
-
if freq:
|
|
191
|
+
if freq := frequency_from_rdf(value):
|
|
198
192
|
dataset.frequency = freq
|
|
199
|
-
elif value in UPDATE_FREQUENCIES:
|
|
200
|
-
dataset.frequency = value
|
|
201
193
|
else:
|
|
202
|
-
|
|
203
|
-
|
|
194
|
+
# FIXME(python 3.12+): prefer `if value in UpdateFrequency`
|
|
195
|
+
try:
|
|
196
|
+
dataset.frequency = UpdateFrequency(value)
|
|
197
|
+
except ValueError:
|
|
198
|
+
dataset.extras["ckan:frequency"] = value
|
|
199
|
+
log.debug("frequency value not handled: %s", value)
|
|
204
200
|
# Temporal coverage start
|
|
205
201
|
elif key == "temporal_start":
|
|
206
202
|
temporal_start = daterange_start(value)
|
udata/harvest/backends/maaf.py
CHANGED
|
@@ -7,6 +7,7 @@ from urllib.parse import urljoin
|
|
|
7
7
|
from lxml import etree, html
|
|
8
8
|
from voluptuous import All, Any, In, Length, Lower, Optional, Schema
|
|
9
9
|
|
|
10
|
+
from udata.core.dataset.constants import UpdateFrequency
|
|
10
11
|
from udata.harvest.backends import BaseBackend
|
|
11
12
|
from udata.harvest.filters import (
|
|
12
13
|
boolean,
|
|
@@ -38,19 +39,19 @@ ZONES = {
|
|
|
38
39
|
|
|
39
40
|
|
|
40
41
|
FREQUENCIES = {
|
|
41
|
-
"ponctuelle":
|
|
42
|
-
"temps réel":
|
|
43
|
-
"quotidienne":
|
|
44
|
-
"hebdomadaire":
|
|
45
|
-
"bimensuelle":
|
|
46
|
-
"mensuelle":
|
|
47
|
-
"bimestrielle":
|
|
48
|
-
"trimestrielle":
|
|
49
|
-
"semestrielle":
|
|
50
|
-
"annuelle":
|
|
51
|
-
"triennale":
|
|
52
|
-
"quinquennale":
|
|
53
|
-
"aucune":
|
|
42
|
+
"ponctuelle": UpdateFrequency.PUNCTUAL,
|
|
43
|
+
"temps réel": UpdateFrequency.CONTINUOUS,
|
|
44
|
+
"quotidienne": UpdateFrequency.DAILY,
|
|
45
|
+
"hebdomadaire": UpdateFrequency.WEEKLY,
|
|
46
|
+
"bimensuelle": UpdateFrequency.SEMIMONTHLY,
|
|
47
|
+
"mensuelle": UpdateFrequency.MONTHLY,
|
|
48
|
+
"bimestrielle": UpdateFrequency.BIMONTHLY,
|
|
49
|
+
"trimestrielle": UpdateFrequency.QUARTERLY,
|
|
50
|
+
"semestrielle": UpdateFrequency.SEMIANNUAL,
|
|
51
|
+
"annuelle": UpdateFrequency.ANNUAL,
|
|
52
|
+
"triennale": UpdateFrequency.TRIENNIAL,
|
|
53
|
+
"quinquennale": UpdateFrequency.QUINQUENNIAL,
|
|
54
|
+
"aucune": UpdateFrequency.UNKNOWN,
|
|
54
55
|
}
|
|
55
56
|
|
|
56
57
|
XSD_PATH = os.path.join(os.path.dirname(__file__), "maaf.xsd")
|
|
@@ -161,7 +162,7 @@ class MaafBackend(BaseBackend):
|
|
|
161
162
|
dataset = self.get_dataset(item.remote_id)
|
|
162
163
|
|
|
163
164
|
dataset.title = metadata["title"]
|
|
164
|
-
dataset.frequency = FREQUENCIES.get(metadata["frequency"],
|
|
165
|
+
dataset.frequency = FREQUENCIES.get(metadata["frequency"], UpdateFrequency.UNKNOWN)
|
|
165
166
|
dataset.description = metadata["notes"]
|
|
166
167
|
dataset.private = metadata["private"]
|
|
167
168
|
dataset.tags = sorted(set(metadata["tags"]))
|
|
@@ -5,6 +5,7 @@ from datetime import date
|
|
|
5
5
|
import pytest
|
|
6
6
|
|
|
7
7
|
from udata.app import create_app
|
|
8
|
+
from udata.core.dataset.constants import UpdateFrequency
|
|
8
9
|
from udata.core.organization.factories import OrganizationFactory
|
|
9
10
|
from udata.core.spatial.factories import GeoZoneFactory
|
|
10
11
|
from udata.harvest import actions
|
|
@@ -336,7 +337,7 @@ def frequency_as_rdf_uri(resource_data):
|
|
|
336
337
|
"resources": [resource_data],
|
|
337
338
|
"extras": [{"key": "frequency", "value": "http://purl.org/cld/freq/daily"}],
|
|
338
339
|
}
|
|
339
|
-
return data, {"expected":
|
|
340
|
+
return data, {"expected": UpdateFrequency.DAILY}
|
|
340
341
|
|
|
341
342
|
|
|
342
343
|
@pytest.fixture
|
|
@@ -348,7 +349,7 @@ def frequency_as_exact_match(resource_data):
|
|
|
348
349
|
"resources": [resource_data],
|
|
349
350
|
"extras": [{"key": "frequency", "value": "daily"}],
|
|
350
351
|
}
|
|
351
|
-
return data, {"expected":
|
|
352
|
+
return data, {"expected": UpdateFrequency.DAILY}
|
|
352
353
|
|
|
353
354
|
|
|
354
355
|
@pytest.fixture
|
|
@@ -486,7 +487,7 @@ def test_can_parse_frequency_as_exact_match(result, kwargs):
|
|
|
486
487
|
|
|
487
488
|
|
|
488
489
|
@pytest.mark.ckan_data("frequency_as_unknown_value")
|
|
489
|
-
def
|
|
490
|
+
def test_can_parse_frequency_as_unknown_value(result, kwargs):
|
|
490
491
|
dataset = dataset_for(result)
|
|
491
492
|
assert dataset.extras["ckan:frequency"] == kwargs["expected"]
|
|
492
493
|
assert dataset.frequency is None
|
|
@@ -10,6 +10,7 @@ from rdflib import Graph
|
|
|
10
10
|
|
|
11
11
|
from udata.core.dataservices.factories import DataserviceFactory
|
|
12
12
|
from udata.core.dataservices.models import Dataservice
|
|
13
|
+
from udata.core.dataset.constants import UpdateFrequency
|
|
13
14
|
from udata.core.dataset.factories import DatasetFactory, LicenseFactory, ResourceSchemaMockData
|
|
14
15
|
from udata.core.dataset.rdf import dataset_from_rdf
|
|
15
16
|
from udata.core.organization.factories import OrganizationFactory
|
|
@@ -560,7 +561,7 @@ class DcatBackendTest:
|
|
|
560
561
|
assert dataset.harvest.issued_at.date() == date(2016, 12, 14)
|
|
561
562
|
assert dataset.harvest.created_at.date() == date(2016, 12, 12)
|
|
562
563
|
assert dataset.harvest.modified_at.date() == date(2016, 12, 14)
|
|
563
|
-
assert dataset.frequency ==
|
|
564
|
+
assert dataset.frequency == UpdateFrequency.DAILY
|
|
564
565
|
assert dataset.description == "Dataset 3 description"
|
|
565
566
|
|
|
566
567
|
assert dataset.temporal_coverage is not None
|
|
@@ -681,7 +682,7 @@ class DcatBackendTest:
|
|
|
681
682
|
dataset = Dataset.objects.filter(organization=org).first()
|
|
682
683
|
|
|
683
684
|
assert dataset is not None
|
|
684
|
-
assert dataset.frequency ==
|
|
685
|
+
assert dataset.frequency == UpdateFrequency.IRREGULAR
|
|
685
686
|
assert "gravi" in dataset.tags # support dcat:keyword
|
|
686
687
|
assert "geodesy" in dataset.tags # support dcat:theme
|
|
687
688
|
assert dataset.license.id == "fr-lo"
|
udata/i18n.py
CHANGED
|
@@ -206,38 +206,13 @@ class I18nBlueprintSetupState(BlueprintSetupState):
|
|
|
206
206
|
if "defaults" in options:
|
|
207
207
|
defaults = dict(defaults, **options.pop("defaults"))
|
|
208
208
|
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
)
|
|
217
|
-
|
|
218
|
-
self.app.add_url_rule(
|
|
219
|
-
rule,
|
|
220
|
-
"%s.%s_redirect" % (self.blueprint.name, endpoint),
|
|
221
|
-
redirect_to_lang,
|
|
222
|
-
defaults=defaults,
|
|
223
|
-
**options,
|
|
224
|
-
)
|
|
225
|
-
else:
|
|
226
|
-
self.app.add_url_rule(
|
|
227
|
-
rule,
|
|
228
|
-
"%s.%s" % (self.blueprint.name, endpoint),
|
|
229
|
-
view_func,
|
|
230
|
-
defaults=defaults,
|
|
231
|
-
**options,
|
|
232
|
-
)
|
|
233
|
-
|
|
234
|
-
self.app.add_url_rule(
|
|
235
|
-
"/<lang:lang_code>" + rule,
|
|
236
|
-
"%s.%s_redirect" % (self.blueprint.name, endpoint),
|
|
237
|
-
redirect_to_unlocalized,
|
|
238
|
-
defaults=defaults,
|
|
239
|
-
**options,
|
|
240
|
-
)
|
|
209
|
+
self.app.add_url_rule(
|
|
210
|
+
rule,
|
|
211
|
+
"%s.%s" % (self.blueprint.name, endpoint),
|
|
212
|
+
view_func,
|
|
213
|
+
defaults=defaults,
|
|
214
|
+
**options,
|
|
215
|
+
)
|
|
241
216
|
|
|
242
217
|
|
|
243
218
|
class I18nBlueprint(Blueprint):
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Convert legacy frequencies to latest vocabulary
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
import logging
|
|
6
|
+
|
|
7
|
+
from udata.core.dataset.constants import UpdateFrequency
|
|
8
|
+
from udata.models import Dataset
|
|
9
|
+
|
|
10
|
+
log = logging.getLogger(__name__)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def migrate(db):
|
|
14
|
+
log.info("Updating datasets legacy frequencies:")
|
|
15
|
+
|
|
16
|
+
for legacy_value, frequency in UpdateFrequency._LEGACY_FREQUENCIES.items():
|
|
17
|
+
count = 0
|
|
18
|
+
for dataset in Dataset.objects(frequency=legacy_value).no_cache().timeout(False):
|
|
19
|
+
# Explicitly call update() to force writing the new frequency string to mongo.
|
|
20
|
+
# We can't rely on save() here because:
|
|
21
|
+
# 1. save() only writes modified fields to mongo, basically comparing the Dataset's
|
|
22
|
+
# object state initially returned by the query with its state when save() is called,
|
|
23
|
+
# and only sending the diffset to mongo.
|
|
24
|
+
# 2. At the ODM layer, the Dataset.frequency field has already been instantiated as an
|
|
25
|
+
# UpdateFrequency object, and so legacy frequency strings have already been
|
|
26
|
+
# mapped to their new ids (via UpdateFrequency._missing_).
|
|
27
|
+
# => While the raw frequency string has changed, the save() function sees the same
|
|
28
|
+
# UpdateFrequency and therefore ignores the field in is diffset.
|
|
29
|
+
dataset.update(frequency=frequency.value)
|
|
30
|
+
# Still call save() afterwards so computed fields like quality_cached are updated if
|
|
31
|
+
# necessary, e.g. if moving from a predictable timedelta to an unpredictable one.
|
|
32
|
+
dataset.save()
|
|
33
|
+
count += 1
|
|
34
|
+
log.info(f"- {legacy_value} -> {frequency.value}: {count} updated")
|
|
35
|
+
|
|
36
|
+
log.info("Completed.")
|
udata/settings.py
CHANGED
|
@@ -101,6 +101,24 @@ class Defaults(object):
|
|
|
101
101
|
SECURITY_RESET_URL = "/reset/"
|
|
102
102
|
SECURITY_CHANGE_EMAIL_URL = "/change-email/"
|
|
103
103
|
|
|
104
|
+
# See https://flask-security.readthedocs.io/en/stable/configuration.html#SECURITY_REDIRECT_BEHAVIOR
|
|
105
|
+
# We do not define all the URLs requested in the documentation because most of the time we do JSON requests in cdata
|
|
106
|
+
# and catch errors instead of followings the redirects.
|
|
107
|
+
# The only place where we don't have control over the redirect is when the user is clicking a link directly to udata
|
|
108
|
+
# (instead of a link to `cdata`) as in /confirm. When the user is clicking on the confirmation link, he's redirected
|
|
109
|
+
# to `confirm_change_email` endpoint, and then udata redirect him to the homepage of `cdata` with a custom flash message.
|
|
110
|
+
SECURITY_REDIRECT_BEHAVIOR = "spa"
|
|
111
|
+
# SECURITY_POST_OAUTH_LOGIN_VIEW = "" # SECURITY_OAUTH_ENABLE is disabled
|
|
112
|
+
# SECURITY_LOGIN_ERROR_VIEW = "" # We don't follow the redirects since we do JSON POST requests during login
|
|
113
|
+
# SECURITY_CONFIRM_ERROR_VIEW = "" # Manually changed in `confirm_change_email`
|
|
114
|
+
# SECURITY_POST_CHANGE_EMAIL_VIEW = "" # We don't follow the redirects since we do JSON POST requests during change email
|
|
115
|
+
# SECURITY_CHANGE_EMAIL_ERROR_VIEW = "" # We don't follow the redirects since we do JSON POST requests during change email
|
|
116
|
+
# SECURITY_POST_CONFIRM_VIEW = "" # Set at runtime. See :SecurityPostConfirmViewAtRuntime
|
|
117
|
+
# SECURITY_RESET_ERROR_VIEW = "" # We don't follow the redirects since we do JSON POST requests during request reset
|
|
118
|
+
# SECURITY_RESET_VIEW = "" # We don't follow the redirects since we do JSON POST requests during request reset
|
|
119
|
+
|
|
120
|
+
SECURITY_SPA_ON_SAME_DOMAIN = False
|
|
121
|
+
|
|
104
122
|
SECURITY_PASSWORD_SALT = "Default uData secret password salt"
|
|
105
123
|
SECURITY_CONFIRM_SALT = "Default uData secret confirm salt"
|
|
106
124
|
SECURITY_RESET_SALT = "Default uData secret reset salt"
|
|
@@ -122,6 +140,15 @@ class Defaults(object):
|
|
|
122
140
|
DAYS_BEFORE_ACCOUNT_INACTIVITY_NOTIFY_DELAY = 30
|
|
123
141
|
MAX_NUMBER_OF_USER_INACTIVITY_NOTIFICATIONS = 200
|
|
124
142
|
|
|
143
|
+
# You can activate CaptchEtat, a captcha.com integration by providing
|
|
144
|
+
# CAPTCHETAT_BASE_URL, CAPTCHETAT_OAUTH_BASE_URL, CAPTCHETAT_CLIENT_ID and CAPTCHETAT_CLIENT_SECRET
|
|
145
|
+
CAPTCHETAT_BASE_URL = None
|
|
146
|
+
CAPTCHETAT_OAUTH_BASE_URL = None
|
|
147
|
+
CAPTCHETAT_CLIENT_ID = None
|
|
148
|
+
CAPTCHETAT_CLIENT_SECRET = None
|
|
149
|
+
CAPTCHETAT_TOKEN_CACHE_KEY = "captchetat-bearer-token"
|
|
150
|
+
CAPTCHETAT_STYLE_NAME = "captchaFR"
|
|
151
|
+
|
|
125
152
|
# Sentry configuration
|
|
126
153
|
SENTRY_DSN = None
|
|
127
154
|
SENTRY_TAGS = {}
|
|
@@ -16,12 +16,10 @@ from udata.app import cache
|
|
|
16
16
|
from udata.core import storages
|
|
17
17
|
from udata.core.badges.factories import badge_factory
|
|
18
18
|
from udata.core.dataset.constants import (
|
|
19
|
-
DEFAULT_FREQUENCY,
|
|
20
19
|
DEFAULT_LICENSE,
|
|
21
20
|
FULL_OBJECTS_HEADER,
|
|
22
|
-
LEGACY_FREQUENCIES,
|
|
23
21
|
RESOURCE_TYPES,
|
|
24
|
-
|
|
22
|
+
UpdateFrequency,
|
|
25
23
|
)
|
|
26
24
|
from udata.core.dataset.factories import (
|
|
27
25
|
CommunityResourceFactory,
|
|
@@ -531,14 +529,14 @@ class DatasetAPITest(APITestCase):
|
|
|
531
529
|
other = GeoLevelFactory(id="other", name="Autre")
|
|
532
530
|
|
|
533
531
|
dataset = DatasetFactory(
|
|
534
|
-
frequency=
|
|
532
|
+
frequency=UpdateFrequency.MONTHLY,
|
|
535
533
|
license=license,
|
|
536
534
|
spatial=SpatialCoverageFactory(zones=[paca.id], granularity=country.id),
|
|
537
535
|
)
|
|
538
536
|
|
|
539
537
|
response = self.get(url_for("apiv2.dataset", dataset=dataset))
|
|
540
538
|
self.assert200(response)
|
|
541
|
-
assert response.json["frequency"] ==
|
|
539
|
+
assert response.json["frequency"] == UpdateFrequency.MONTHLY.id
|
|
542
540
|
assert response.json["license"] == "lov2"
|
|
543
541
|
assert response.json["spatial"]["zones"][0] == paca.id
|
|
544
542
|
assert response.json["spatial"]["granularity"] == "country"
|
|
@@ -550,8 +548,8 @@ class DatasetAPITest(APITestCase):
|
|
|
550
548
|
},
|
|
551
549
|
)
|
|
552
550
|
self.assert200(response)
|
|
553
|
-
assert response.json["frequency"]["id"] ==
|
|
554
|
-
assert response.json["frequency"]["label"] ==
|
|
551
|
+
assert response.json["frequency"]["id"] == UpdateFrequency.MONTHLY.id
|
|
552
|
+
assert response.json["frequency"]["label"] == UpdateFrequency.MONTHLY.label
|
|
555
553
|
assert response.json["license"]["id"] == "lov2"
|
|
556
554
|
assert response.json["license"]["title"] == license.title
|
|
557
555
|
assert response.json["spatial"]["zones"][0]["id"] == paca.id
|
|
@@ -572,8 +570,8 @@ class DatasetAPITest(APITestCase):
|
|
|
572
570
|
},
|
|
573
571
|
)
|
|
574
572
|
self.assert200(response)
|
|
575
|
-
assert response.json["frequency"]["id"] ==
|
|
576
|
-
assert response.json["frequency"]["label"] ==
|
|
573
|
+
assert response.json["frequency"]["id"] == UpdateFrequency.UNKNOWN.id
|
|
574
|
+
assert response.json["frequency"]["label"] == UpdateFrequency.UNKNOWN.label
|
|
577
575
|
assert response.json["license"]["id"] == DEFAULT_LICENSE["id"]
|
|
578
576
|
assert response.json["license"]["title"] == DEFAULT_LICENSE["title"]
|
|
579
577
|
assert len(response.json["spatial"]["zones"]) == 0
|
|
@@ -723,12 +721,12 @@ class DatasetAPITest(APITestCase):
|
|
|
723
721
|
"""It should create a dataset from the API with a legacy frequency"""
|
|
724
722
|
self.login()
|
|
725
723
|
|
|
726
|
-
for oldFreq, newFreq in
|
|
724
|
+
for oldFreq, newFreq in UpdateFrequency._LEGACY_FREQUENCIES.items(): # type: ignore[misc]
|
|
727
725
|
data = DatasetFactory.as_dict()
|
|
728
726
|
data["frequency"] = oldFreq
|
|
729
727
|
response = self.post(url_for("api.datasets"), data)
|
|
730
728
|
self.assert201(response)
|
|
731
|
-
self.assertEqual(response.json["frequency"], newFreq)
|
|
729
|
+
self.assertEqual(response.json["frequency"], newFreq.id)
|
|
732
730
|
|
|
733
731
|
def test_dataset_api_update(self):
|
|
734
732
|
"""It should update a dataset from the API"""
|
|
@@ -741,6 +739,37 @@ class DatasetAPITest(APITestCase):
|
|
|
741
739
|
self.assertEqual(Dataset.objects.count(), 1)
|
|
742
740
|
self.assertEqual(Dataset.objects.first().description, "new description")
|
|
743
741
|
|
|
742
|
+
def test_dataset_api_update_valid_frequency(self):
|
|
743
|
+
"""It should update a dataset from the API"""
|
|
744
|
+
user = self.login()
|
|
745
|
+
dataset = DatasetFactory(owner=user)
|
|
746
|
+
data = dataset.to_dict()
|
|
747
|
+
data["frequency"] = "monthly"
|
|
748
|
+
response = self.put(url_for("api.dataset", dataset=dataset), data)
|
|
749
|
+
self.assert200(response)
|
|
750
|
+
self.assertEqual(Dataset.objects.count(), 1)
|
|
751
|
+
self.assertEqual(Dataset.objects.first().frequency, UpdateFrequency.MONTHLY)
|
|
752
|
+
|
|
753
|
+
def test_dataset_api_update_invalid_frequency(self):
|
|
754
|
+
"""It should return an error saying the frequency is invalid"""
|
|
755
|
+
user = self.login()
|
|
756
|
+
dataset = DatasetFactory(owner=user, frequency=UpdateFrequency.ANNUAL)
|
|
757
|
+
data = dataset.to_dict()
|
|
758
|
+
|
|
759
|
+
data["frequency"] = 1 # invalid type
|
|
760
|
+
response = self.put(url_for("api.dataset", dataset=dataset), data)
|
|
761
|
+
self.assert400(response)
|
|
762
|
+
self.assertEqual(response.json.get("message"), "'1' is not a valid UpdateFrequency")
|
|
763
|
+
self.assertEqual(Dataset.objects.count(), 1)
|
|
764
|
+
self.assertEqual(Dataset.objects.first().frequency, UpdateFrequency.ANNUAL)
|
|
765
|
+
|
|
766
|
+
data["frequency"] = "foo" # valid type but invalid term
|
|
767
|
+
response = self.put(url_for("api.dataset", dataset=dataset), data)
|
|
768
|
+
self.assert400(response)
|
|
769
|
+
self.assertEqual(response.json.get("message"), "'foo' is not a valid UpdateFrequency")
|
|
770
|
+
self.assertEqual(Dataset.objects.count(), 1)
|
|
771
|
+
self.assertEqual(Dataset.objects.first().frequency, UpdateFrequency.ANNUAL)
|
|
772
|
+
|
|
744
773
|
def test_cannot_modify_dataset_id(self):
|
|
745
774
|
user = self.login()
|
|
746
775
|
dataset = DatasetFactory(owner=user)
|
|
@@ -2044,7 +2073,7 @@ class DatasetReferencesAPITest(APITestCase):
|
|
|
2044
2073
|
"""It should fetch the dataset frequencies list from the API"""
|
|
2045
2074
|
response = self.get(url_for("api.dataset_frequencies"))
|
|
2046
2075
|
self.assert200(response)
|
|
2047
|
-
self.assertEqual(len(response.json), len(
|
|
2076
|
+
self.assertEqual(len(response.json), len(UpdateFrequency))
|
|
2048
2077
|
|
|
2049
2078
|
def test_dataset_allowed_resources_extensions(self):
|
|
2050
2079
|
"""It should fetch the resources allowed extensions list from the API"""
|
|
@@ -20,7 +20,7 @@ from udata.core.dataset.activities import (
|
|
|
20
20
|
UserUpdatedDataset,
|
|
21
21
|
UserUpdatedResource,
|
|
22
22
|
)
|
|
23
|
-
from udata.core.dataset.constants import
|
|
23
|
+
from udata.core.dataset.constants import UpdateFrequency
|
|
24
24
|
from udata.core.dataset.exceptions import (
|
|
25
25
|
SchemasCacheUnavailableException,
|
|
26
26
|
SchemasCatalogNotFoundException,
|
|
@@ -157,49 +157,13 @@ class DatasetModelTest:
|
|
|
157
157
|
dataset = DatasetFactory()
|
|
158
158
|
assert dataset.next_update is None
|
|
159
159
|
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
assert_equal_dates(dataset.next_update, datetime.utcnow() + timedelta(hours=1))
|
|
163
|
-
|
|
164
|
-
@pytest.mark.parametrize("freq", ["fourTimesADay", "threeTimesADay", "semidaily", "daily"])
|
|
165
|
-
def test_next_update_daily(self, freq):
|
|
166
|
-
dataset = DatasetFactory(frequency=freq)
|
|
167
|
-
assert_equal_dates(dataset.next_update, datetime.utcnow() + timedelta(days=1))
|
|
168
|
-
|
|
169
|
-
@pytest.mark.parametrize("freq", ["fourTimesAWeek", "threeTimesAWeek", "semiweekly", "weekly"])
|
|
170
|
-
def test_next_update_weekly(self, freq):
|
|
171
|
-
dataset = DatasetFactory(frequency=freq)
|
|
172
|
-
assert_equal_dates(dataset.next_update, datetime.utcnow() + timedelta(days=7))
|
|
173
|
-
|
|
174
|
-
def test_next_update_biweekly(self):
|
|
175
|
-
dataset = DatasetFactory(frequency="biweekly")
|
|
176
|
-
assert_equal_dates(dataset.next_update, datetime.utcnow() + timedelta(weeks=2))
|
|
177
|
-
|
|
178
|
-
def test_next_update_quarterly(self):
|
|
179
|
-
dataset = DatasetFactory(frequency="quarterly")
|
|
180
|
-
assert_equal_dates(dataset.next_update, datetime.utcnow() + timedelta(days=365 / 4))
|
|
181
|
-
|
|
182
|
-
@pytest.mark.parametrize("freq", ["threeTimesAYear", "semiannual", "annual"])
|
|
183
|
-
def test_next_update_annual(self, freq):
|
|
184
|
-
dataset = DatasetFactory(frequency=freq)
|
|
185
|
-
assert_equal_dates(dataset.next_update, datetime.utcnow() + timedelta(days=365))
|
|
186
|
-
|
|
187
|
-
def test_next_update_biennial(self):
|
|
188
|
-
dataset = DatasetFactory(frequency="biennial")
|
|
189
|
-
assert_equal_dates(dataset.next_update, datetime.utcnow() + timedelta(days=365 * 2))
|
|
190
|
-
|
|
191
|
-
def test_next_update_triennial(self):
|
|
192
|
-
dataset = DatasetFactory(frequency="triennial")
|
|
193
|
-
assert_equal_dates(dataset.next_update, datetime.utcnow() + timedelta(days=365 * 3))
|
|
194
|
-
|
|
195
|
-
def test_next_update_quinquennial(self):
|
|
196
|
-
dataset = DatasetFactory(frequency="quinquennial")
|
|
197
|
-
assert_equal_dates(dataset.next_update, datetime.utcnow() + timedelta(days=365 * 5))
|
|
198
|
-
|
|
199
|
-
@pytest.mark.parametrize("freq", ["continuous", "punctual", "irregular", "unknown"])
|
|
200
|
-
def test_next_update_undefined(self, freq):
|
|
160
|
+
@pytest.mark.parametrize("freq", list(UpdateFrequency) + [None])
|
|
161
|
+
def test_next_update(self, freq: UpdateFrequency | None):
|
|
201
162
|
dataset = DatasetFactory(frequency=freq)
|
|
202
|
-
|
|
163
|
+
if freq is None or freq.delta is None:
|
|
164
|
+
assert dataset.next_update is None
|
|
165
|
+
else:
|
|
166
|
+
assert_equal_dates(dataset.next_update, freq.next_update(datetime.utcnow()))
|
|
203
167
|
|
|
204
168
|
def test_quality_default(self):
|
|
205
169
|
dataset = DatasetFactory(description="")
|
|
@@ -212,21 +176,21 @@ class DatasetModelTest:
|
|
|
212
176
|
"score": 0,
|
|
213
177
|
}
|
|
214
178
|
|
|
215
|
-
@pytest.mark.parametrize("freq",
|
|
216
|
-
def test_quality_frequency_update(self, freq):
|
|
179
|
+
@pytest.mark.parametrize("freq", list(UpdateFrequency) + [None])
|
|
180
|
+
def test_quality_frequency_update(self, freq: UpdateFrequency | None):
|
|
217
181
|
dataset = DatasetFactory(description="", frequency=freq)
|
|
218
|
-
if freq
|
|
182
|
+
if freq in [None, UpdateFrequency.UNKNOWN]:
|
|
219
183
|
assert dataset.quality["update_frequency"] is False
|
|
220
184
|
assert "update_fulfilled_in_time" not in dataset.quality
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
185
|
+
else:
|
|
186
|
+
assert dataset.quality["update_frequency"] is True
|
|
187
|
+
assert dataset.quality["update_fulfilled_in_time"] is True
|
|
188
|
+
assert dataset.quality["score"] == Dataset.normalize_score(2)
|
|
225
189
|
|
|
226
190
|
def test_quality_frequency_update_one_day_late(self):
|
|
227
191
|
dataset = DatasetFactory(
|
|
228
192
|
description="",
|
|
229
|
-
frequency=
|
|
193
|
+
frequency=UpdateFrequency.DAILY,
|
|
230
194
|
last_modified_internal=datetime.utcnow() - timedelta(days=1, hours=1),
|
|
231
195
|
)
|
|
232
196
|
assert dataset.quality["update_frequency"] is True
|
|
@@ -236,7 +200,7 @@ class DatasetModelTest:
|
|
|
236
200
|
def test_quality_frequency_update_two_days_late(self):
|
|
237
201
|
dataset = DatasetFactory(
|
|
238
202
|
description="",
|
|
239
|
-
frequency=
|
|
203
|
+
frequency=UpdateFrequency.DAILY,
|
|
240
204
|
last_modified_internal=datetime.utcnow() - timedelta(days=2, hours=1),
|
|
241
205
|
)
|
|
242
206
|
assert dataset.quality["update_frequency"] is True
|
|
@@ -309,7 +273,7 @@ class DatasetModelTest:
|
|
|
309
273
|
assert dataset.tags[1] == "this-is-a-tag"
|
|
310
274
|
|
|
311
275
|
def test_legacy_frequencies(self):
|
|
312
|
-
for oldFreq, newFreq in
|
|
276
|
+
for oldFreq, newFreq in UpdateFrequency._LEGACY_FREQUENCIES.items(): # type: ignore[misc]
|
|
313
277
|
dataset = DatasetFactory(frequency=oldFreq)
|
|
314
278
|
assert dataset.frequency == newFreq
|
|
315
279
|
|