kinto 18.1.0__py3-none-any.whl → 20.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of kinto might be problematic. Click here for more details.
- kinto/__init__.py +1 -0
- kinto/__main__.py +1 -19
- kinto/config/kinto.tpl +5 -15
- kinto/contribute.json +27 -0
- kinto/core/__init__.py +21 -8
- kinto/core/cornice/__init__.py +93 -0
- kinto/core/cornice/cors.py +144 -0
- kinto/core/cornice/errors.py +40 -0
- kinto/core/cornice/pyramidhook.py +373 -0
- kinto/core/cornice/renderer.py +89 -0
- kinto/core/cornice/resource.py +205 -0
- kinto/core/cornice/service.py +641 -0
- kinto/core/cornice/util.py +138 -0
- kinto/core/cornice/validators/__init__.py +94 -0
- kinto/core/cornice/validators/_colander.py +142 -0
- kinto/core/cornice/validators/_marshmallow.py +182 -0
- kinto/core/cornice_swagger/__init__.py +92 -0
- kinto/core/cornice_swagger/converters/__init__.py +21 -0
- kinto/core/cornice_swagger/converters/exceptions.py +6 -0
- kinto/core/cornice_swagger/converters/parameters.py +90 -0
- kinto/core/cornice_swagger/converters/schema.py +249 -0
- kinto/core/cornice_swagger/swagger.py +725 -0
- kinto/core/cornice_swagger/templates/index.html +73 -0
- kinto/core/cornice_swagger/templates/index_script_template.html +21 -0
- kinto/core/cornice_swagger/util.py +42 -0
- kinto/core/cornice_swagger/views.py +78 -0
- kinto/core/errors.py +6 -4
- kinto/core/initialization.py +129 -59
- kinto/core/metrics.py +93 -0
- kinto/core/openapi.py +2 -3
- kinto/core/permission/memory.py +3 -2
- kinto/core/permission/postgresql/__init__.py +9 -9
- kinto/core/permission/testing.py +6 -0
- kinto/core/resource/__init__.py +9 -4
- kinto/core/resource/schema.py +1 -2
- kinto/core/resource/viewset.py +1 -1
- kinto/core/statsd.py +1 -63
- kinto/core/storage/__init__.py +15 -0
- kinto/core/storage/memory.py +20 -3
- kinto/core/storage/postgresql/__init__.py +31 -1
- kinto/core/storage/postgresql/client.py +2 -2
- kinto/core/storage/postgresql/migrations/migration_022_023.sql +5 -0
- kinto/core/storage/postgresql/pool.py +1 -1
- kinto/core/storage/postgresql/schema.sql +3 -2
- kinto/core/storage/testing.py +41 -1
- kinto/core/testing.py +6 -2
- kinto/core/utils.py +14 -4
- kinto/core/views/batch.py +1 -1
- kinto/core/views/errors.py +4 -3
- kinto/core/views/openapi.py +1 -1
- kinto/plugins/accounts/__init__.py +3 -21
- kinto/plugins/accounts/authentication.py +8 -54
- kinto/plugins/accounts/utils.py +0 -133
- kinto/plugins/accounts/{views/__init__.py → views.py} +7 -62
- kinto/plugins/admin/VERSION +1 -1
- kinto/plugins/admin/build/VERSION +1 -0
- kinto/plugins/admin/build/assets/asn1-EdZsLKOL.js +1 -0
- kinto/plugins/admin/build/assets/clojure-BMjYHr_A.js +1 -0
- kinto/plugins/admin/build/assets/css-BnMrqG3P.js +1 -0
- kinto/plugins/admin/build/assets/index-Cs7JVwIg.css +6 -0
- kinto/plugins/admin/build/assets/index-CylsivYB.js +165 -0
- kinto/plugins/admin/build/assets/javascript-qCveANmP.js +1 -0
- kinto/plugins/admin/build/assets/logo-VBRiKSPX.png +0 -0
- kinto/plugins/admin/build/assets/mllike-CXdrOF99.js +1 -0
- kinto/plugins/admin/build/assets/python-BuPzkPfP.js +1 -0
- kinto/plugins/admin/build/assets/rpm-CTu-6PCP.js +1 -0
- kinto/plugins/admin/build/assets/sql-D0XecflT.js +1 -0
- kinto/plugins/admin/build/assets/ttcn-cfg-B9xdYoR4.js +1 -0
- kinto/plugins/admin/build/index.html +18 -0
- kinto/plugins/default_bucket/__init__.py +1 -2
- kinto/plugins/flush.py +2 -2
- kinto/plugins/history/__init__.py +15 -6
- kinto/plugins/history/listener.py +68 -5
- kinto/plugins/openid/views.py +1 -1
- kinto/plugins/prometheus.py +203 -0
- kinto/plugins/statsd.py +78 -0
- kinto/views/contribute.py +14 -13
- {kinto-18.1.0.dist-info → kinto-20.4.0.dist-info}/METADATA +31 -32
- kinto-20.4.0.dist-info/RECORD +149 -0
- {kinto-18.1.0.dist-info → kinto-20.4.0.dist-info}/WHEEL +1 -1
- kinto/plugins/accounts/mails.py +0 -96
- kinto/plugins/accounts/views/validation.py +0 -136
- kinto/plugins/quotas/__init__.py +0 -22
- kinto/plugins/quotas/listener.py +0 -226
- kinto/plugins/quotas/scripts.py +0 -80
- kinto/plugins/quotas/utils.py +0 -7
- kinto/scripts.py +0 -41
- kinto-18.1.0.dist-info/RECORD +0 -116
- {kinto-18.1.0.dist-info → kinto-20.4.0.dist-info}/entry_points.txt +0 -0
- {kinto-18.1.0.dist-info → kinto-20.4.0.dist-info/licenses}/LICENSE +0 -0
- {kinto-18.1.0.dist-info → kinto-20.4.0.dist-info}/top_level.txt +0 -0
kinto/core/permission/testing.py
CHANGED
|
@@ -344,6 +344,12 @@ class PermissionTest:
|
|
|
344
344
|
)
|
|
345
345
|
self.assertEqual(sorted(per_object_ids.keys()), ["/url/a", "/url/a/id/1", "/url/a/id/2"])
|
|
346
346
|
|
|
347
|
+
def test_accessible_objects_with_user_principle(self):
|
|
348
|
+
self.permission.add_user_principal("user1", "group")
|
|
349
|
+
self.permission.add_principal_to_ace("id1", "write", "user1")
|
|
350
|
+
per_object_ids = self.permission.get_accessible_objects(["user1"])
|
|
351
|
+
self.assertEqual(sorted(per_object_ids.keys()), ["id1"])
|
|
352
|
+
|
|
347
353
|
#
|
|
348
354
|
# get_object_permissions()
|
|
349
355
|
#
|
kinto/core/resource/__init__.py
CHANGED
|
@@ -665,7 +665,7 @@ class Resource:
|
|
|
665
665
|
obj = self._get_object_or_404(self.object_id)
|
|
666
666
|
self._raise_412_if_modified(obj)
|
|
667
667
|
|
|
668
|
-
#
|
|
668
|
+
# Retrieve the last_modified information from a querystring if present.
|
|
669
669
|
last_modified = self.request.validated["querystring"].get("last_modified")
|
|
670
670
|
|
|
671
671
|
# If less or equal than current object. Ignore it.
|
|
@@ -1058,6 +1058,11 @@ class Resource:
|
|
|
1058
1058
|
|
|
1059
1059
|
def _extract_filters(self):
|
|
1060
1060
|
"""Extracts filters from QueryString parameters."""
|
|
1061
|
+
|
|
1062
|
+
def is_valid_timestamp(value):
|
|
1063
|
+
# Is either integer, or integer as string, or integer between 2 quotes.
|
|
1064
|
+
return isinstance(value, int) or re.match(r'^(\d+)$|^("\d+")$', str(value))
|
|
1065
|
+
|
|
1061
1066
|
queryparams = self.request.validated["querystring"]
|
|
1062
1067
|
|
|
1063
1068
|
filters = []
|
|
@@ -1081,7 +1086,7 @@ class Resource:
|
|
|
1081
1086
|
operator = COMPARISON.GT
|
|
1082
1087
|
else:
|
|
1083
1088
|
if param == "_to":
|
|
1084
|
-
message = "_to is now deprecated,
|
|
1089
|
+
message = "_to is now deprecated, you should use _before instead"
|
|
1085
1090
|
url = (
|
|
1086
1091
|
"https://kinto.readthedocs.io/en/2.4.0/api/"
|
|
1087
1092
|
"resource.html#list-of-available-url-"
|
|
@@ -1090,7 +1095,7 @@ class Resource:
|
|
|
1090
1095
|
send_alert(self.request, message, url)
|
|
1091
1096
|
operator = COMPARISON.LT
|
|
1092
1097
|
|
|
1093
|
-
if value
|
|
1098
|
+
if value is not None and not is_valid_timestamp(value):
|
|
1094
1099
|
raise_invalid(self.request, **error_details)
|
|
1095
1100
|
|
|
1096
1101
|
filters.append(Filter(self.model.modified_field, value, operator))
|
|
@@ -1127,7 +1132,7 @@ class Resource:
|
|
|
1127
1132
|
error_details["description"] = "Invalid character 0x00"
|
|
1128
1133
|
raise_invalid(self.request, **error_details)
|
|
1129
1134
|
|
|
1130
|
-
if field == self.model.modified_field and value
|
|
1135
|
+
if field == self.model.modified_field and not is_valid_timestamp(value):
|
|
1131
1136
|
raise_invalid(self.request, **error_details)
|
|
1132
1137
|
|
|
1133
1138
|
filters.append(Filter(field, value, operator))
|
kinto/core/resource/schema.py
CHANGED
|
@@ -37,8 +37,7 @@ class URL(URL):
|
|
|
37
37
|
|
|
38
38
|
def __init__(self, *args, **kwargs):
|
|
39
39
|
message = (
|
|
40
|
-
"`kinto.core.resource.schema.URL` is deprecated, "
|
|
41
|
-
"use `kinto.core.schema.URL` instead."
|
|
40
|
+
"`kinto.core.resource.schema.URL` is deprecated, use `kinto.core.schema.URL` instead."
|
|
42
41
|
)
|
|
43
42
|
warnings.warn(message, DeprecationWarning)
|
|
44
43
|
super().__init__(*args, **kwargs)
|
kinto/core/resource/viewset.py
CHANGED
|
@@ -2,10 +2,10 @@ import functools
|
|
|
2
2
|
import warnings
|
|
3
3
|
|
|
4
4
|
import colander
|
|
5
|
-
from cornice.validators import colander_validator
|
|
6
5
|
from pyramid.settings import asbool
|
|
7
6
|
|
|
8
7
|
from kinto.core import authorization
|
|
8
|
+
from kinto.core.cornice.validators import colander_validator
|
|
9
9
|
|
|
10
10
|
from .schema import (
|
|
11
11
|
ObjectGetQuerySchema,
|
kinto/core/statsd.py
CHANGED
|
@@ -1,63 +1 @@
|
|
|
1
|
-
import
|
|
2
|
-
from urllib.parse import urlparse
|
|
3
|
-
|
|
4
|
-
from pyramid.exceptions import ConfigurationError
|
|
5
|
-
|
|
6
|
-
from kinto.core import utils
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
try:
|
|
10
|
-
import statsd as statsd_module
|
|
11
|
-
except ImportError: # pragma: no cover
|
|
12
|
-
statsd_module = None
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
class Client:
|
|
16
|
-
def __init__(self, host, port, prefix):
|
|
17
|
-
self._client = statsd_module.StatsClient(host, port, prefix=prefix)
|
|
18
|
-
|
|
19
|
-
def watch_execution_time(self, obj, prefix="", classname=None):
|
|
20
|
-
classname = classname or utils.classname(obj)
|
|
21
|
-
members = dir(obj)
|
|
22
|
-
for name in members:
|
|
23
|
-
value = getattr(obj, name)
|
|
24
|
-
is_method = isinstance(value, types.MethodType)
|
|
25
|
-
if not name.startswith("_") and is_method:
|
|
26
|
-
statsd_key = f"{prefix}.{classname}.{name}"
|
|
27
|
-
decorated_method = self.timer(statsd_key)(value)
|
|
28
|
-
setattr(obj, name, decorated_method)
|
|
29
|
-
|
|
30
|
-
def timer(self, key):
|
|
31
|
-
return self._client.timer(key)
|
|
32
|
-
|
|
33
|
-
def count(self, key, count=1, unique=None):
|
|
34
|
-
if unique is None:
|
|
35
|
-
return self._client.incr(key, count=count)
|
|
36
|
-
else:
|
|
37
|
-
return self._client.set(key, unique)
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
def statsd_count(request, count_key):
|
|
41
|
-
statsd = request.registry.statsd
|
|
42
|
-
if statsd:
|
|
43
|
-
statsd.count(count_key)
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
def load_from_config(config):
|
|
47
|
-
# If this is called, it means that a ``statsd_url`` was specified in settings.
|
|
48
|
-
# (see ``kinto.core.initialization``)
|
|
49
|
-
# Raise a proper error if the ``statsd`` module is not installed.
|
|
50
|
-
if statsd_module is None:
|
|
51
|
-
error_msg = "Please install Kinto with monitoring dependencies (e.g. statsd package)"
|
|
52
|
-
raise ConfigurationError(error_msg)
|
|
53
|
-
|
|
54
|
-
settings = config.get_settings()
|
|
55
|
-
uri = settings["statsd_url"]
|
|
56
|
-
uri = urlparse(uri)
|
|
57
|
-
|
|
58
|
-
if settings["project_name"] != "":
|
|
59
|
-
prefix = settings["project_name"]
|
|
60
|
-
else:
|
|
61
|
-
prefix = settings["statsd_prefix"]
|
|
62
|
-
|
|
63
|
-
return Client(uri.hostname, uri.port, prefix)
|
|
1
|
+
from kinto.plugins.statsd import load_from_config # noqa: F401
|
kinto/core/storage/__init__.py
CHANGED
|
@@ -87,6 +87,21 @@ class StorageBase:
|
|
|
87
87
|
"""
|
|
88
88
|
raise NotImplementedError
|
|
89
89
|
|
|
90
|
+
def all_resources_timestamps(self, resource_name):
|
|
91
|
+
"""Get the highest timestamp of every objects in this `resource_name` for
|
|
92
|
+
each `parent_id`.
|
|
93
|
+
|
|
94
|
+
.. note::
|
|
95
|
+
|
|
96
|
+
This should take deleted objects into account.
|
|
97
|
+
|
|
98
|
+
:param str resource_name: the resource name.
|
|
99
|
+
|
|
100
|
+
:returns: the latest timestamp of the resource by `parent_id`.
|
|
101
|
+
:rtype: dict[str, int]
|
|
102
|
+
"""
|
|
103
|
+
raise NotImplementedError
|
|
104
|
+
|
|
90
105
|
def create(
|
|
91
106
|
self,
|
|
92
107
|
resource_name,
|
kinto/core/storage/memory.py
CHANGED
|
@@ -153,6 +153,10 @@ class Storage(MemoryBasedStorage):
|
|
|
153
153
|
raise exceptions.ReadonlyError(message=error_msg)
|
|
154
154
|
return self.bump_and_store_timestamp(resource_name, parent_id)
|
|
155
155
|
|
|
156
|
+
@synchronized
|
|
157
|
+
def all_resources_timestamps(self, resource_name):
|
|
158
|
+
return {k: v[resource_name] for k, v in self._timestamps.items() if resource_name in v}
|
|
159
|
+
|
|
156
160
|
def bump_and_store_timestamp(
|
|
157
161
|
self, resource_name, parent_id, obj=None, modified_field=None, last_modified=None
|
|
158
162
|
):
|
|
@@ -284,13 +288,26 @@ class Storage(MemoryBasedStorage):
|
|
|
284
288
|
modified_field=DEFAULT_MODIFIED_FIELD,
|
|
285
289
|
):
|
|
286
290
|
parent_id_match = re.compile(parent_id.replace("*", ".*"))
|
|
287
|
-
|
|
291
|
+
|
|
292
|
+
timestamps_by_parent_id = {
|
|
288
293
|
pid: resources
|
|
289
|
-
for pid, resources in self.
|
|
294
|
+
for pid, resources in self._timestamps.items()
|
|
290
295
|
if parent_id_match.match(pid)
|
|
291
296
|
}
|
|
297
|
+
if resource_name is not None:
|
|
298
|
+
for pid, resources in timestamps_by_parent_id.items():
|
|
299
|
+
del self._timestamps[pid][resource_name]
|
|
300
|
+
else:
|
|
301
|
+
for pid, resources in timestamps_by_parent_id.items():
|
|
302
|
+
del self._timestamps[pid]
|
|
303
|
+
|
|
292
304
|
num_deleted = 0
|
|
293
|
-
|
|
305
|
+
tombstones_by_parent_id = {
|
|
306
|
+
pid: resources
|
|
307
|
+
for pid, resources in self._cemetery.items()
|
|
308
|
+
if parent_id_match.match(pid)
|
|
309
|
+
}
|
|
310
|
+
for pid, resources in tombstones_by_parent_id.items():
|
|
294
311
|
if resource_name is not None:
|
|
295
312
|
resources = {resource_name: resources[resource_name]}
|
|
296
313
|
for resource, resource_objects in resources.items():
|
|
@@ -79,7 +79,7 @@ class Storage(StorageBase, MigratorMixin):
|
|
|
79
79
|
|
|
80
80
|
# MigratorMixin attributes.
|
|
81
81
|
name = "storage"
|
|
82
|
-
schema_version =
|
|
82
|
+
schema_version = 23
|
|
83
83
|
schema_file = os.path.join(HERE, "schema.sql")
|
|
84
84
|
migrations_directory = os.path.join(HERE, "migrations")
|
|
85
85
|
|
|
@@ -247,6 +247,36 @@ class Storage(StorageBase, MigratorMixin):
|
|
|
247
247
|
|
|
248
248
|
return obj.last_epoch
|
|
249
249
|
|
|
250
|
+
def all_resources_timestamps(self, resource_name):
|
|
251
|
+
query = """
|
|
252
|
+
WITH existing_timestamps AS (
|
|
253
|
+
-- Timestamp of latest object by parent_id.
|
|
254
|
+
(
|
|
255
|
+
SELECT parent_id, MAX(last_modified) AS last_modified
|
|
256
|
+
FROM objects
|
|
257
|
+
WHERE resource_name = :resource_name
|
|
258
|
+
GROUP BY parent_id
|
|
259
|
+
)
|
|
260
|
+
-- Timestamp of resources without sub-objects.
|
|
261
|
+
UNION
|
|
262
|
+
(
|
|
263
|
+
SELECT parent_id, last_modified
|
|
264
|
+
FROM timestamps
|
|
265
|
+
WHERE resource_name = :resource_name
|
|
266
|
+
)
|
|
267
|
+
)
|
|
268
|
+
SELECT parent_id, MAX(as_epoch(last_modified)) AS last_modified
|
|
269
|
+
FROM existing_timestamps
|
|
270
|
+
GROUP BY parent_id
|
|
271
|
+
ORDER BY last_modified DESC
|
|
272
|
+
"""
|
|
273
|
+
with self.client.connect(readonly=True) as conn:
|
|
274
|
+
result = conn.execute(sa.text(query), dict(resource_name=resource_name))
|
|
275
|
+
rows = result.fetchmany(self._max_fetch_size + 1)
|
|
276
|
+
|
|
277
|
+
results = {r[0]: r[1] for r in rows}
|
|
278
|
+
return results
|
|
279
|
+
|
|
250
280
|
@deprecate_kwargs({"collection_id": "resource_name", "record": "obj"})
|
|
251
281
|
def create(
|
|
252
282
|
self,
|
|
@@ -95,14 +95,14 @@ def create_from_config(config, prefix="", with_transaction=True):
|
|
|
95
95
|
url = filtered_settings[prefix + "url"]
|
|
96
96
|
existing_client = _CLIENTS[transaction_per_request].get(url)
|
|
97
97
|
if existing_client:
|
|
98
|
-
msg = "Reuse existing PostgreSQL connection.
|
|
98
|
+
msg = f"Reuse existing PostgreSQL connection. Parameters {prefix}* will be ignored."
|
|
99
99
|
warnings.warn(msg)
|
|
100
100
|
return existing_client
|
|
101
101
|
|
|
102
102
|
# Initialize SQLAlchemy engine from filtered_settings.
|
|
103
103
|
poolclass_key = prefix + "poolclass"
|
|
104
104
|
filtered_settings.setdefault(
|
|
105
|
-
poolclass_key, ("kinto.core.storage.postgresql.
|
|
105
|
+
poolclass_key, ("kinto.core.storage.postgresql.pool.QueuePoolWithMaxBacklog")
|
|
106
106
|
)
|
|
107
107
|
filtered_settings[poolclass_key] = config.maybe_dotted(filtered_settings[poolclass_key])
|
|
108
108
|
engine = sqlalchemy.engine_from_config(filtered_settings, prefix=prefix, url=url)
|
|
@@ -47,7 +47,8 @@ CREATE UNIQUE INDEX IF NOT EXISTS idx_objects_parent_id_resource_name_last_modif
|
|
|
47
47
|
ON objects(parent_id, resource_name, last_modified DESC);
|
|
48
48
|
CREATE INDEX IF NOT EXISTS idx_objects_last_modified_epoch
|
|
49
49
|
ON objects(as_epoch(last_modified));
|
|
50
|
-
|
|
50
|
+
CREATE INDEX IF NOT EXISTS idx_objects_resource_name_parent_id_deleted
|
|
51
|
+
ON objects(resource_name, parent_id, deleted);
|
|
51
52
|
|
|
52
53
|
CREATE TABLE IF NOT EXISTS timestamps (
|
|
53
54
|
parent_id TEXT NOT NULL COLLATE "C",
|
|
@@ -131,4 +132,4 @@ INSERT INTO metadata (name, value) VALUES ('created_at', NOW()::TEXT);
|
|
|
131
132
|
|
|
132
133
|
-- Set storage schema version.
|
|
133
134
|
-- Should match ``kinto.core.storage.postgresql.PostgreSQL.schema_version``
|
|
134
|
-
INSERT INTO metadata (name, value) VALUES ('storage_schema_version', '
|
|
135
|
+
INSERT INTO metadata (name, value) VALUES ('storage_schema_version', '23');
|
kinto/core/storage/testing.py
CHANGED
|
@@ -783,6 +783,39 @@ class TimestampsTest:
|
|
|
783
783
|
after = self.storage.resource_timestamp(**self.storage_kw)
|
|
784
784
|
self.assertTrue(before < after)
|
|
785
785
|
|
|
786
|
+
def test_all_timestamps_by_parent_id(self):
|
|
787
|
+
self.storage.create(obj={"id": "main"}, resource_name="bucket", parent_id="")
|
|
788
|
+
self.storage.create(obj={"id": "cid1"}, resource_name="collection", parent_id="/main")
|
|
789
|
+
self.storage.create(obj={"id": "cid2"}, resource_name="collection", parent_id="/main")
|
|
790
|
+
self.storage.create(obj={}, resource_name="record", parent_id="/main/cid2")
|
|
791
|
+
self.storage.create(obj={}, resource_name="record", parent_id="/main/cid2")
|
|
792
|
+
|
|
793
|
+
self.assertEqual(
|
|
794
|
+
{
|
|
795
|
+
"": self.storage.resource_timestamp(resource_name="bucket", parent_id=""),
|
|
796
|
+
},
|
|
797
|
+
self.storage.all_resources_timestamps(resource_name="bucket"),
|
|
798
|
+
)
|
|
799
|
+
self.assertEqual(
|
|
800
|
+
{
|
|
801
|
+
"/main": self.storage.resource_timestamp(
|
|
802
|
+
resource_name="collection", parent_id="/main"
|
|
803
|
+
),
|
|
804
|
+
},
|
|
805
|
+
self.storage.all_resources_timestamps(resource_name="collection"),
|
|
806
|
+
)
|
|
807
|
+
self.assertEqual(
|
|
808
|
+
{
|
|
809
|
+
"/main/cid1": self.storage.resource_timestamp(
|
|
810
|
+
resource_name="record", parent_id="/main/cid1"
|
|
811
|
+
),
|
|
812
|
+
"/main/cid2": self.storage.resource_timestamp(
|
|
813
|
+
resource_name="record", parent_id="/main/cid2"
|
|
814
|
+
),
|
|
815
|
+
},
|
|
816
|
+
self.storage.all_resources_timestamps(resource_name="record"),
|
|
817
|
+
)
|
|
818
|
+
|
|
786
819
|
@skip_if_ci
|
|
787
820
|
def test_timestamps_are_unique(self): # pragma: no cover
|
|
788
821
|
obtained = []
|
|
@@ -1263,6 +1296,9 @@ class DeletedObjectsTest:
|
|
|
1263
1296
|
self.create_object(parent_id="/abc/a", resource_name="c")
|
|
1264
1297
|
self.create_object(parent_id="/efg", resource_name="c")
|
|
1265
1298
|
|
|
1299
|
+
all_timestamps = self.storage.all_resources_timestamps(resource_name="c")
|
|
1300
|
+
self.assertEqual(set(all_timestamps.keys()), {"/abc/a", "/efg"})
|
|
1301
|
+
|
|
1266
1302
|
before1 = self.storage.resource_timestamp(parent_id="/abc/a", resource_name="c")
|
|
1267
1303
|
# Different parent_id with object.
|
|
1268
1304
|
before2 = self.storage.resource_timestamp(parent_id="/efg", resource_name="c")
|
|
@@ -1272,11 +1308,15 @@ class DeletedObjectsTest:
|
|
|
1272
1308
|
self.storage.delete_all(parent_id="/abc/*", resource_name=None, with_deleted=False)
|
|
1273
1309
|
self.storage.purge_deleted(parent_id="/abc/*", resource_name=None)
|
|
1274
1310
|
|
|
1311
|
+
all_timestamps = self.storage.all_resources_timestamps(resource_name="c")
|
|
1312
|
+
self.assertEqual(set(all_timestamps.keys()), {"/efg", "/ijk"})
|
|
1313
|
+
|
|
1314
|
+
time.sleep(0.002) # make sure we don't recreate timestamps at same msec.
|
|
1275
1315
|
after1 = self.storage.resource_timestamp(parent_id="/abc/a", resource_name="c")
|
|
1276
1316
|
after2 = self.storage.resource_timestamp(parent_id="/efg", resource_name="c")
|
|
1277
1317
|
after3 = self.storage.resource_timestamp(parent_id="/ijk", resource_name="c")
|
|
1278
1318
|
|
|
1279
|
-
self.assertNotEqual(before1, after1)
|
|
1319
|
+
self.assertNotEqual(before1, after1) # timestamp was removed, it will differ.
|
|
1280
1320
|
self.assertEqual(before2, after2)
|
|
1281
1321
|
self.assertEqual(before3, after3)
|
|
1282
1322
|
|
kinto/core/testing.py
CHANGED
|
@@ -5,18 +5,22 @@ from collections import defaultdict
|
|
|
5
5
|
from unittest import mock
|
|
6
6
|
|
|
7
7
|
import webtest
|
|
8
|
-
from cornice import errors as cornice_errors
|
|
9
8
|
from pyramid.url import parse_url_overrides
|
|
10
9
|
|
|
11
|
-
from kinto.core import DEFAULT_SETTINGS
|
|
10
|
+
from kinto.core import DEFAULT_SETTINGS
|
|
11
|
+
from kinto.core.cornice import errors as cornice_errors
|
|
12
12
|
from kinto.core.storage import generators
|
|
13
13
|
from kinto.core.utils import encode64, follow_subrequest, memcache, sqlalchemy
|
|
14
|
+
from kinto.plugins import prometheus, statsd
|
|
14
15
|
|
|
15
16
|
|
|
16
17
|
skip_if_ci = unittest.skipIf("CI" in os.environ, "ci")
|
|
17
18
|
skip_if_no_postgresql = unittest.skipIf(sqlalchemy is None, "postgresql is not installed.")
|
|
18
19
|
skip_if_no_memcached = unittest.skipIf(memcache is None, "memcached is not installed.")
|
|
19
20
|
skip_if_no_statsd = unittest.skipIf(not statsd.statsd_module, "statsd is not installed.")
|
|
21
|
+
skip_if_no_prometheus = unittest.skipIf(
|
|
22
|
+
not prometheus.prometheus_module, "prometheus is not installed."
|
|
23
|
+
)
|
|
20
24
|
|
|
21
25
|
|
|
22
26
|
class DummyRequest(mock.MagicMock):
|
kinto/core/utils.py
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import collections.abc as collections_abc
|
|
2
|
+
import functools
|
|
2
3
|
import hashlib
|
|
3
4
|
import hmac
|
|
4
5
|
import os
|
|
@@ -12,7 +13,6 @@ from urllib.parse import unquote
|
|
|
12
13
|
import jsonpatch
|
|
13
14
|
import rapidjson
|
|
14
15
|
from colander import null
|
|
15
|
-
from cornice import cors
|
|
16
16
|
from pyramid import httpexceptions
|
|
17
17
|
from pyramid.authorization import Authenticated
|
|
18
18
|
from pyramid.interfaces import IRoutesMapper
|
|
@@ -20,6 +20,8 @@ from pyramid.request import Request, apply_request_extensions
|
|
|
20
20
|
from pyramid.settings import aslist
|
|
21
21
|
from pyramid.view import render_view_to_response
|
|
22
22
|
|
|
23
|
+
from kinto.core.cornice import cors
|
|
24
|
+
|
|
23
25
|
|
|
24
26
|
try:
|
|
25
27
|
import sqlalchemy
|
|
@@ -261,8 +263,9 @@ def reapply_cors(request, response):
|
|
|
261
263
|
settings = request.registry.settings
|
|
262
264
|
allowed_origins = set(aslist(settings["cors_origins"]))
|
|
263
265
|
required_origins = {"*", origin}
|
|
264
|
-
|
|
265
|
-
|
|
266
|
+
matches = allowed_origins.intersection(required_origins)
|
|
267
|
+
if matches:
|
|
268
|
+
response.headers["Access-Control-Allow-Origin"] = matches.pop()
|
|
266
269
|
|
|
267
270
|
# Import service here because kinto.core import utils
|
|
268
271
|
from kinto.core import Service
|
|
@@ -287,7 +290,7 @@ def current_service(request):
|
|
|
287
290
|
"""Return the Cornice service matching the specified request.
|
|
288
291
|
|
|
289
292
|
:returns: the service or None if unmatched.
|
|
290
|
-
:rtype: cornice.Service
|
|
293
|
+
:rtype: kinto.core.cornice.Service
|
|
291
294
|
"""
|
|
292
295
|
if request.matched_route:
|
|
293
296
|
services = request.registry.cornice_services
|
|
@@ -541,3 +544,10 @@ def apply_json_patch(obj, ops):
|
|
|
541
544
|
raise ValueError(e)
|
|
542
545
|
|
|
543
546
|
return result
|
|
547
|
+
|
|
548
|
+
|
|
549
|
+
def safe_wraps(wrapper, *args, **kwargs):
|
|
550
|
+
"""Safely wraps partial functions."""
|
|
551
|
+
while isinstance(wrapper, functools.partial):
|
|
552
|
+
wrapper = wrapper.func
|
|
553
|
+
return functools.wraps(wrapper, *args, **kwargs)
|
kinto/core/views/batch.py
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import logging
|
|
2
2
|
|
|
3
3
|
import colander
|
|
4
|
-
from cornice.validators import colander_validator
|
|
5
4
|
from pyramid import httpexceptions
|
|
6
5
|
from pyramid.security import NO_PERMISSION_REQUIRED
|
|
7
6
|
|
|
8
7
|
from kinto.core import Service, errors
|
|
8
|
+
from kinto.core.cornice.validators import colander_validator
|
|
9
9
|
from kinto.core.errors import ErrorSchema
|
|
10
10
|
from kinto.core.resource.viewset import CONTENT_TYPES
|
|
11
11
|
from kinto.core.utils import build_request, build_response, merge_dicts
|
kinto/core/views/errors.py
CHANGED
|
@@ -22,6 +22,8 @@ def authorization_required(response, request):
|
|
|
22
22
|
"""
|
|
23
23
|
if Authenticated not in request.effective_principals:
|
|
24
24
|
if response.content_type != "application/json":
|
|
25
|
+
# This is always the case when `HTTPForbidden` is raised by Pyramid
|
|
26
|
+
# on protected views with unauthenticated requests.
|
|
25
27
|
error_msg = "Please authenticate yourself to use this endpoint."
|
|
26
28
|
response = http_error(
|
|
27
29
|
httpexceptions.HTTPUnauthorized(),
|
|
@@ -53,7 +55,7 @@ def page_not_found(response, request):
|
|
|
53
55
|
|
|
54
56
|
if not request.path.startswith(f"/{request.registry.route_prefix}"):
|
|
55
57
|
errno = ERRORS.VERSION_NOT_AVAILABLE
|
|
56
|
-
error_msg = "The requested API version is not available
|
|
58
|
+
error_msg = "The requested API version is not available on this server."
|
|
57
59
|
elif trailing_slash_redirection_enabled:
|
|
58
60
|
redirect = None
|
|
59
61
|
|
|
@@ -80,8 +82,7 @@ def page_not_found(response, request):
|
|
|
80
82
|
def service_unavailable(response, request):
|
|
81
83
|
if response.content_type != "application/json":
|
|
82
84
|
error_msg = (
|
|
83
|
-
"Service temporary unavailable "
|
|
84
|
-
"due to overloading or maintenance, please retry later."
|
|
85
|
+
"Service temporary unavailable due to overloading or maintenance, please retry later."
|
|
85
86
|
)
|
|
86
87
|
response = http_error(response, errno=ERRORS.BACKEND, message=error_msg)
|
|
87
88
|
|
kinto/core/views/openapi.py
CHANGED
|
@@ -6,19 +6,12 @@ from pyramid.exceptions import ConfigurationError
|
|
|
6
6
|
from kinto.authorization import PERMISSIONS_INHERITANCE_TREE
|
|
7
7
|
|
|
8
8
|
from .authentication import AccountsAuthenticationPolicy as AccountsPolicy
|
|
9
|
-
from .utils import
|
|
10
|
-
ACCOUNT_CACHE_KEY,
|
|
11
|
-
ACCOUNT_POLICY_NAME,
|
|
12
|
-
ACCOUNT_RESET_PASSWORD_CACHE_KEY,
|
|
13
|
-
ACCOUNT_VALIDATION_CACHE_KEY,
|
|
14
|
-
)
|
|
9
|
+
from .utils import ACCOUNT_CACHE_KEY, ACCOUNT_POLICY_NAME
|
|
15
10
|
|
|
16
11
|
|
|
17
12
|
__all__ = [
|
|
18
13
|
"ACCOUNT_CACHE_KEY",
|
|
19
14
|
"ACCOUNT_POLICY_NAME",
|
|
20
|
-
"ACCOUNT_RESET_PASSWORD_CACHE_KEY",
|
|
21
|
-
"ACCOUNT_VALIDATION_CACHE_KEY",
|
|
22
15
|
"AccountsPolicy",
|
|
23
16
|
]
|
|
24
17
|
|
|
@@ -27,16 +20,13 @@ DOCS_URL = "https://kinto.readthedocs.io/en/stable/api/1.x/accounts.html"
|
|
|
27
20
|
|
|
28
21
|
def includeme(config):
|
|
29
22
|
settings = config.get_settings()
|
|
30
|
-
validation_enabled = settings.get("account_validation", False)
|
|
31
23
|
config.add_api_capability(
|
|
32
24
|
"accounts",
|
|
33
25
|
description="Manage user accounts.",
|
|
34
26
|
url="https://kinto.readthedocs.io/en/latest/api/1.x/accounts.html",
|
|
35
|
-
validation_enabled=
|
|
27
|
+
validation_enabled=False,
|
|
36
28
|
)
|
|
37
29
|
kwargs = {}
|
|
38
|
-
if not validation_enabled:
|
|
39
|
-
kwargs["ignore"] = "kinto.plugins.accounts.views.validation"
|
|
40
30
|
config.scan("kinto.plugins.accounts.views", **kwargs)
|
|
41
31
|
|
|
42
32
|
PERMISSIONS_INHERITANCE_TREE["root"].update({"account:create": {}})
|
|
@@ -45,13 +35,6 @@ def includeme(config):
|
|
|
45
35
|
"read": {"account": ["write", "read"]},
|
|
46
36
|
}
|
|
47
37
|
|
|
48
|
-
if validation_enabled:
|
|
49
|
-
# Valid mailers other than the default are `debug` and `testing`
|
|
50
|
-
# according to
|
|
51
|
-
# https://docs.pylonsproject.org/projects/pyramid_mailer/en/latest/#debugging
|
|
52
|
-
mailer = settings.get("mail.mailer", "")
|
|
53
|
-
config.include("pyramid_mailer" + (f".{mailer}" if mailer else ""))
|
|
54
|
-
|
|
55
38
|
# Check that the account policy is mentioned in config if included.
|
|
56
39
|
accountClass = "AccountsPolicy"
|
|
57
40
|
policy = None
|
|
@@ -83,8 +66,7 @@ def includeme(config):
|
|
|
83
66
|
if "basicauth" in auth_policies and policy in auth_policies:
|
|
84
67
|
if auth_policies.index("basicauth") < auth_policies.index(policy):
|
|
85
68
|
error_msg = (
|
|
86
|
-
"'basicauth' should not be mentioned before '%s' "
|
|
87
|
-
"in 'multiauth.policies' setting."
|
|
69
|
+
"'basicauth' should not be mentioned before '%s' in 'multiauth.policies' setting."
|
|
88
70
|
) % policy
|
|
89
71
|
raise ConfigurationError(error_msg)
|
|
90
72
|
|
|
@@ -5,31 +5,27 @@ from kinto.core import utils
|
|
|
5
5
|
from kinto.core.storage import exceptions as storage_exceptions
|
|
6
6
|
|
|
7
7
|
from .utils import (
|
|
8
|
+
ACCOUNT_CACHE_KEY,
|
|
8
9
|
ACCOUNT_POLICY_NAME,
|
|
9
|
-
cache_account,
|
|
10
|
-
delete_cached_reset_password,
|
|
11
|
-
get_account_cache_key,
|
|
12
|
-
get_cached_account,
|
|
13
|
-
get_cached_reset_password,
|
|
14
|
-
is_validated,
|
|
15
|
-
refresh_cached_account,
|
|
16
10
|
)
|
|
17
11
|
|
|
18
12
|
|
|
19
13
|
def account_check(username, password, request):
|
|
20
14
|
settings = request.registry.settings
|
|
21
|
-
|
|
22
|
-
cache_key =
|
|
15
|
+
hmac_secret = settings["userid_hmac_secret"]
|
|
16
|
+
cache_key = utils.hmac_digest(hmac_secret, ACCOUNT_CACHE_KEY.format(username))
|
|
17
|
+
cache_ttl = int(settings.get("account_cache_ttl_seconds", 30))
|
|
23
18
|
hashed_password = utils.hmac_digest(cache_key, password)
|
|
24
19
|
|
|
25
20
|
# Check cache to see whether somebody has recently logged in with the same
|
|
26
21
|
# username and password.
|
|
27
|
-
|
|
22
|
+
cache = request.registry.cache
|
|
23
|
+
cache_result = cache.get(cache_key)
|
|
28
24
|
|
|
29
25
|
# Username and password have been verified previously. No need to compare hashes
|
|
30
26
|
if cache_result == hashed_password:
|
|
31
27
|
# Refresh the cache TTL.
|
|
32
|
-
|
|
28
|
+
cache.expire(cache_key, cache_ttl)
|
|
33
29
|
return True
|
|
34
30
|
|
|
35
31
|
# Back to standard procedure
|
|
@@ -41,53 +37,11 @@ def account_check(username, password, request):
|
|
|
41
37
|
except storage_exceptions.ObjectNotFoundError:
|
|
42
38
|
return None
|
|
43
39
|
|
|
44
|
-
if validation_enabled and not is_validated(existing):
|
|
45
|
-
return None
|
|
46
|
-
|
|
47
40
|
hashed = existing["password"].encode(encoding="utf-8")
|
|
48
41
|
pwd_str = password.encode(encoding="utf-8")
|
|
49
42
|
# Check if password is valid (it is a very expensive computation)
|
|
50
43
|
if bcrypt.checkpw(pwd_str, hashed):
|
|
51
|
-
|
|
52
|
-
return True
|
|
53
|
-
|
|
54
|
-
# Last chance, is this a "reset password" flow?
|
|
55
|
-
return reset_password_flow(username, password, request)
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
def reset_password_flow(username, password, request):
|
|
59
|
-
cache_key = get_account_cache_key(username, request.registry)
|
|
60
|
-
hashed_password = utils.hmac_digest(cache_key, password)
|
|
61
|
-
pwd_str = password.encode(encoding="utf-8")
|
|
62
|
-
|
|
63
|
-
cached_password = get_cached_reset_password(username, request.registry)
|
|
64
|
-
if not cached_password:
|
|
65
|
-
return None
|
|
66
|
-
|
|
67
|
-
# The temporary reset password is only available for changing a user's password.
|
|
68
|
-
if request.method.lower() not in ["post", "put", "patch"]:
|
|
69
|
-
return None
|
|
70
|
-
|
|
71
|
-
# Only allow modifying a user account, no other resource.
|
|
72
|
-
uri = utils.strip_uri_prefix(request.path)
|
|
73
|
-
resource_name, _ = utils.view_lookup(request, uri)
|
|
74
|
-
if resource_name != "account":
|
|
75
|
-
return None
|
|
76
|
-
|
|
77
|
-
try:
|
|
78
|
-
data = request.json["data"]
|
|
79
|
-
except (ValueError, KeyError):
|
|
80
|
-
return None
|
|
81
|
-
|
|
82
|
-
# Request one and only one data field: the `password`.
|
|
83
|
-
if not data or "password" not in data or len(data.keys()) > 1:
|
|
84
|
-
return None
|
|
85
|
-
|
|
86
|
-
cached_password_str = cached_password.encode(encoding="utf-8")
|
|
87
|
-
if bcrypt.checkpw(pwd_str, cached_password_str):
|
|
88
|
-
# Remove the temporary reset password from the cache.
|
|
89
|
-
delete_cached_reset_password(username, request.registry)
|
|
90
|
-
cache_account(hashed_password, username, request.registry)
|
|
44
|
+
cache.set(cache_key, hashed_password, ttl=cache_ttl)
|
|
91
45
|
return True
|
|
92
46
|
|
|
93
47
|
|