kinto 18.1.0__py3-none-any.whl → 19.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of kinto might be problematic. Click here for more details.
- kinto/__init__.py +1 -0
- kinto/__main__.py +1 -2
- kinto/core/__init__.py +4 -5
- kinto/core/initialization.py +135 -54
- kinto/core/metrics.py +93 -0
- kinto/core/permission/postgresql/__init__.py +9 -9
- kinto/core/resource/__init__.py +9 -4
- kinto/core/resource/schema.py +1 -2
- kinto/core/statsd.py +1 -63
- kinto/core/storage/postgresql/client.py +2 -2
- kinto/core/testing.py +5 -1
- kinto/core/utils.py +11 -2
- kinto/core/views/errors.py +2 -3
- kinto/plugins/accounts/__init__.py +1 -2
- kinto/plugins/admin/VERSION +1 -1
- kinto/plugins/admin/build/VERSION +1 -0
- kinto/plugins/admin/build/assets/asn1-CGOzndHr.js +1 -0
- kinto/plugins/admin/build/assets/clojure-BMjYHr_A.js +1 -0
- kinto/plugins/admin/build/assets/css-BnMrqG3P.js +1 -0
- kinto/plugins/admin/build/assets/index-BKIg2XW8.js +165 -0
- kinto/plugins/admin/build/assets/index-D8oiN37x.css +6 -0
- kinto/plugins/admin/build/assets/javascript-iSgyE4tI.js +1 -0
- kinto/plugins/admin/build/assets/logo-VBRiKSPX.png +0 -0
- kinto/plugins/admin/build/assets/mllike-C_8OmSiT.js +1 -0
- kinto/plugins/admin/build/assets/python-BuPzkPfP.js +1 -0
- kinto/plugins/admin/build/assets/rpm-CTu-6PCP.js +1 -0
- kinto/plugins/admin/build/assets/sql-C4g8LzGK.js +1 -0
- kinto/plugins/admin/build/assets/ttcn-cfg-BIkV9KBc.js +1 -0
- kinto/plugins/admin/build/index.html +18 -0
- kinto/plugins/default_bucket/__init__.py +1 -2
- kinto/plugins/flush.py +1 -1
- kinto/plugins/history/__init__.py +5 -6
- kinto/plugins/prometheus.py +203 -0
- kinto/plugins/quotas/__init__.py +6 -7
- kinto/plugins/statsd.py +78 -0
- {kinto-18.1.0.dist-info → kinto-19.4.0.dist-info}/METADATA +31 -30
- {kinto-18.1.0.dist-info → kinto-19.4.0.dist-info}/RECORD +41 -24
- {kinto-18.1.0.dist-info → kinto-19.4.0.dist-info}/WHEEL +1 -1
- {kinto-18.1.0.dist-info → kinto-19.4.0.dist-info}/LICENSE +0 -0
- {kinto-18.1.0.dist-info → kinto-19.4.0.dist-info}/entry_points.txt +0 -0
- {kinto-18.1.0.dist-info → kinto-19.4.0.dist-info}/top_level.txt +0 -0
kinto/__init__.py
CHANGED
kinto/__main__.py
CHANGED
|
@@ -161,8 +161,7 @@ def main(args=None):
|
|
|
161
161
|
if not backend:
|
|
162
162
|
while True:
|
|
163
163
|
prompt = (
|
|
164
|
-
"Select the backend you would like to use: "
|
|
165
|
-
"(1 - postgresql, default - memory) "
|
|
164
|
+
"Select the backend you would like to use: (1 - postgresql, default - memory) "
|
|
166
165
|
)
|
|
167
166
|
answer = input(prompt).strip()
|
|
168
167
|
try:
|
kinto/core/__init__.py
CHANGED
|
@@ -66,8 +66,8 @@ DEFAULT_SETTINGS = {
|
|
|
66
66
|
"kinto.core.initialization.setup_authentication",
|
|
67
67
|
"kinto.core.initialization.setup_backoff",
|
|
68
68
|
"kinto.core.initialization.setup_sentry",
|
|
69
|
-
"kinto.core.initialization.setup_statsd",
|
|
70
69
|
"kinto.core.initialization.setup_listeners",
|
|
70
|
+
"kinto.core.initialization.setup_metrics",
|
|
71
71
|
"kinto.core.events.setup_transaction_hook",
|
|
72
72
|
),
|
|
73
73
|
"event_listeners": "",
|
|
@@ -96,6 +96,7 @@ DEFAULT_SETTINGS = {
|
|
|
96
96
|
"statsd_backend": "kinto.core.statsd",
|
|
97
97
|
"statsd_prefix": "kinto.core",
|
|
98
98
|
"statsd_url": None,
|
|
99
|
+
"metrics_matchdict_fields": [],
|
|
99
100
|
"storage_backend": "",
|
|
100
101
|
"storage_url": "",
|
|
101
102
|
"storage_max_fetch_size": 10000,
|
|
@@ -109,10 +110,8 @@ DEFAULT_SETTINGS = {
|
|
|
109
110
|
"trailing_slash_redirect_ttl_seconds": 3600,
|
|
110
111
|
"multiauth.groupfinder": "kinto.core.authorization.groupfinder",
|
|
111
112
|
"multiauth.policies": "",
|
|
112
|
-
"multiauth.policy.basicauth.use":
|
|
113
|
-
|
|
114
|
-
),
|
|
115
|
-
"multiauth.authorization_policy": ("kinto.core.authorization." "AuthorizationPolicy"),
|
|
113
|
+
"multiauth.policy.basicauth.use": "kinto.core.authentication.BasicAuthAuthenticationPolicy",
|
|
114
|
+
"multiauth.authorization_policy": "kinto.core.authorization.AuthorizationPolicy",
|
|
116
115
|
}
|
|
117
116
|
|
|
118
117
|
|
kinto/core/initialization.py
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import logging
|
|
2
2
|
import random
|
|
3
3
|
import re
|
|
4
|
+
import urllib.parse
|
|
4
5
|
import warnings
|
|
5
6
|
from datetime import datetime
|
|
6
7
|
from secrets import token_hex
|
|
@@ -21,7 +22,7 @@ from pyramid.security import NO_PERMISSION_REQUIRED
|
|
|
21
22
|
from pyramid.settings import asbool, aslist
|
|
22
23
|
from pyramid_multiauth import MultiAuthenticationPolicy, MultiAuthPolicySelected
|
|
23
24
|
|
|
24
|
-
from kinto.core import cache, errors, permission, storage, utils
|
|
25
|
+
from kinto.core import cache, errors, metrics, permission, storage, utils
|
|
25
26
|
from kinto.core.events import ACTIONS, ResourceChanged, ResourceRead
|
|
26
27
|
|
|
27
28
|
|
|
@@ -212,7 +213,7 @@ def setup_deprecation(config):
|
|
|
212
213
|
|
|
213
214
|
def _end_of_life_tween_factory(handler, registry):
|
|
214
215
|
"""Pyramid tween to handle service end of life."""
|
|
215
|
-
deprecation_msg = "The service you are trying to connect no longer exists
|
|
216
|
+
deprecation_msg = "The service you are trying to connect no longer exists at this location."
|
|
216
217
|
|
|
217
218
|
def eos_tween(request):
|
|
218
219
|
eos_date = registry.settings["eos"]
|
|
@@ -334,51 +335,13 @@ def setup_sentry(config):
|
|
|
334
335
|
|
|
335
336
|
|
|
336
337
|
def setup_statsd(config):
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
config.registry.statsd = client
|
|
346
|
-
|
|
347
|
-
client.watch_execution_time(config.registry.cache, prefix="backend")
|
|
348
|
-
client.watch_execution_time(config.registry.storage, prefix="backend")
|
|
349
|
-
client.watch_execution_time(config.registry.permission, prefix="backend")
|
|
350
|
-
|
|
351
|
-
# Commit so that configured policy can be queried.
|
|
352
|
-
config.commit()
|
|
353
|
-
policy = config.registry.queryUtility(IAuthenticationPolicy)
|
|
354
|
-
if isinstance(policy, MultiAuthenticationPolicy):
|
|
355
|
-
for name, subpolicy in policy.get_policies():
|
|
356
|
-
client.watch_execution_time(subpolicy, prefix="authentication", classname=name)
|
|
357
|
-
else:
|
|
358
|
-
client.watch_execution_time(policy, prefix="authentication")
|
|
359
|
-
|
|
360
|
-
def on_new_response(event):
|
|
361
|
-
request = event.request
|
|
362
|
-
|
|
363
|
-
# Count unique users.
|
|
364
|
-
user_id = request.prefixed_userid
|
|
365
|
-
if user_id:
|
|
366
|
-
# Get rid of colons in metric packet (see #1282).
|
|
367
|
-
user_id = user_id.replace(":", ".")
|
|
368
|
-
client.count("users", unique=user_id)
|
|
369
|
-
|
|
370
|
-
# Count authentication verifications.
|
|
371
|
-
if hasattr(request, "authn_type"):
|
|
372
|
-
client.count(f"authn_type.{request.authn_type}")
|
|
373
|
-
|
|
374
|
-
# Count view calls.
|
|
375
|
-
service = request.current_service
|
|
376
|
-
if service:
|
|
377
|
-
client.count(f"view.{service.name}.{request.method}")
|
|
378
|
-
|
|
379
|
-
config.add_subscriber(on_new_response, NewResponse)
|
|
380
|
-
|
|
381
|
-
return client
|
|
338
|
+
# It would be pretty rare to find users that have a custom ``kinto.initialization_sequence`` setting.
|
|
339
|
+
# But just in case, warn that it will be removed in next major.
|
|
340
|
+
warnings.warn(
|
|
341
|
+
"``setup_statsd()`` is now deprecated. Use ``kinto.core.initialization.setup_metrics()`` instead.",
|
|
342
|
+
DeprecationWarning,
|
|
343
|
+
)
|
|
344
|
+
setup_metrics(config)
|
|
382
345
|
|
|
383
346
|
|
|
384
347
|
def install_middlewares(app, settings):
|
|
@@ -466,6 +429,126 @@ def setup_logging(config):
|
|
|
466
429
|
config.add_subscriber(on_new_response, NewResponse)
|
|
467
430
|
|
|
468
431
|
|
|
432
|
+
def setup_metrics(config):
|
|
433
|
+
settings = config.get_settings()
|
|
434
|
+
|
|
435
|
+
# Register a no-op metrics service by default.
|
|
436
|
+
config.registry.registerUtility(metrics.NoOpMetricsService(), metrics.IMetricsService)
|
|
437
|
+
|
|
438
|
+
# This does not fully respect the Pyramid/ZCA patterns, but the rest of Kinto uses
|
|
439
|
+
# `registry.storage`, `registry.cache`, etc. Consistency seems more important.
|
|
440
|
+
config.registry.__class__.metrics = property(
|
|
441
|
+
lambda reg: reg.queryUtility(metrics.IMetricsService)
|
|
442
|
+
)
|
|
443
|
+
|
|
444
|
+
def deprecated_registry(self):
|
|
445
|
+
warnings.warn(
|
|
446
|
+
"``config.registry.statsd`` is now deprecated. Use ``config.registry.metrics`` instead.",
|
|
447
|
+
DeprecationWarning,
|
|
448
|
+
)
|
|
449
|
+
return self.metrics
|
|
450
|
+
|
|
451
|
+
config.registry.__class__.statsd = property(deprecated_registry)
|
|
452
|
+
|
|
453
|
+
def on_app_created(event):
|
|
454
|
+
config = event.app
|
|
455
|
+
metrics_service = config.registry.metrics
|
|
456
|
+
|
|
457
|
+
metrics.watch_execution_time(metrics_service, config.registry.cache, prefix="backend")
|
|
458
|
+
metrics.watch_execution_time(metrics_service, config.registry.storage, prefix="backend")
|
|
459
|
+
metrics.watch_execution_time(metrics_service, config.registry.permission, prefix="backend")
|
|
460
|
+
|
|
461
|
+
policy = config.registry.queryUtility(IAuthenticationPolicy)
|
|
462
|
+
if isinstance(policy, MultiAuthenticationPolicy):
|
|
463
|
+
for name, subpolicy in policy.get_policies():
|
|
464
|
+
metrics.watch_execution_time(
|
|
465
|
+
metrics_service, subpolicy, prefix="authentication", classname=name
|
|
466
|
+
)
|
|
467
|
+
else:
|
|
468
|
+
metrics.watch_execution_time(metrics_service, policy, prefix="authentication")
|
|
469
|
+
|
|
470
|
+
config.add_subscriber(on_app_created, ApplicationCreated)
|
|
471
|
+
|
|
472
|
+
def on_new_response(event):
|
|
473
|
+
request = event.request
|
|
474
|
+
metrics_service = config.registry.metrics
|
|
475
|
+
|
|
476
|
+
try:
|
|
477
|
+
endpoint = utils.strip_uri_prefix(request.path)
|
|
478
|
+
endpoint = urllib.parse.quote_plus(endpoint, safe="/?&=-_")
|
|
479
|
+
except UnicodeDecodeError as e:
|
|
480
|
+
# This `on_new_response` callback is also called when a HTTP 400
|
|
481
|
+
# is returned because of an invalid UTF-8 path. We still want metrics.
|
|
482
|
+
endpoint = str(e)
|
|
483
|
+
|
|
484
|
+
# Count unique users.
|
|
485
|
+
user_id = request.prefixed_userid
|
|
486
|
+
if user_id:
|
|
487
|
+
# Get rid of colons in metric packet (see #1282).
|
|
488
|
+
auth, user_id = user_id.split(":")
|
|
489
|
+
metrics_service.count("users", unique=[("auth", auth), ("userid", user_id)])
|
|
490
|
+
|
|
491
|
+
# Add extra labels to metrics, based on fields extracted from the request matchdict.
|
|
492
|
+
metrics_matchdict_fields = aslist(settings["metrics_matchdict_fields"])
|
|
493
|
+
# Turn the `id` field of object endpoints into `{resource}_id` (eg. `mushroom_id`, `bucket_id`)
|
|
494
|
+
enhanced_matchdict = dict(request.matchdict or {})
|
|
495
|
+
try:
|
|
496
|
+
enhanced_matchdict[request.current_resource_name + "_id"] = enhanced_matchdict.get(
|
|
497
|
+
"id", ""
|
|
498
|
+
)
|
|
499
|
+
except AttributeError:
|
|
500
|
+
# Not on a resource.
|
|
501
|
+
pass
|
|
502
|
+
metrics_matchdict_labels = [
|
|
503
|
+
(field, enhanced_matchdict.get(field, "")) for field in metrics_matchdict_fields
|
|
504
|
+
]
|
|
505
|
+
|
|
506
|
+
# Count served requests.
|
|
507
|
+
metrics_service.count(
|
|
508
|
+
"request_summary",
|
|
509
|
+
unique=[
|
|
510
|
+
("method", request.method.lower()),
|
|
511
|
+
("endpoint", endpoint),
|
|
512
|
+
("status", str(event.response.status_code)),
|
|
513
|
+
]
|
|
514
|
+
+ metrics_matchdict_labels,
|
|
515
|
+
)
|
|
516
|
+
|
|
517
|
+
try:
|
|
518
|
+
current = utils.msec_time()
|
|
519
|
+
duration = current - request._received_at
|
|
520
|
+
metrics_service.observe(
|
|
521
|
+
"request_duration",
|
|
522
|
+
duration,
|
|
523
|
+
labels=[("endpoint", endpoint)] + metrics_matchdict_labels,
|
|
524
|
+
)
|
|
525
|
+
except AttributeError: # pragma: no cover
|
|
526
|
+
# Logging was not setup in this Kinto app (unlikely but possible)
|
|
527
|
+
pass
|
|
528
|
+
|
|
529
|
+
# Observe response size.
|
|
530
|
+
metrics_service.observe(
|
|
531
|
+
"request_size",
|
|
532
|
+
len(event.response.body or b""),
|
|
533
|
+
labels=[("endpoint", endpoint)] + metrics_matchdict_labels,
|
|
534
|
+
)
|
|
535
|
+
|
|
536
|
+
# Count authentication verifications.
|
|
537
|
+
if hasattr(request, "authn_type"):
|
|
538
|
+
metrics_service.count(f"authn_type.{request.authn_type}")
|
|
539
|
+
|
|
540
|
+
# Count view calls.
|
|
541
|
+
service = request.current_service
|
|
542
|
+
if service:
|
|
543
|
+
metrics_service.count(f"view.{service.name}.{request.method}")
|
|
544
|
+
|
|
545
|
+
config.add_subscriber(on_new_response, NewResponse)
|
|
546
|
+
|
|
547
|
+
# While statsd is deprecated, we include its plugin by default for retro-compability.
|
|
548
|
+
if settings["statsd_url"]:
|
|
549
|
+
config.include("kinto.plugins.statsd")
|
|
550
|
+
|
|
551
|
+
|
|
469
552
|
class EventActionFilter:
|
|
470
553
|
def __init__(self, actions, config):
|
|
471
554
|
actions = ACTIONS.from_string_list(actions)
|
|
@@ -518,11 +601,9 @@ def setup_listeners(config):
|
|
|
518
601
|
listener_mod = config.maybe_dotted(module_value)
|
|
519
602
|
listener = listener_mod.load_from_config(config, prefix)
|
|
520
603
|
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
key = f"listeners.{name}"
|
|
525
|
-
listener = statsd_client.timer(key)(listener.__call__)
|
|
604
|
+
wrapped_listener = metrics.listener_with_timer(
|
|
605
|
+
config, f"listeners.{name}", listener.__call__
|
|
606
|
+
)
|
|
526
607
|
|
|
527
608
|
# Optional filter by event action.
|
|
528
609
|
actions_setting = prefix + "actions"
|
|
@@ -548,11 +629,11 @@ def setup_listeners(config):
|
|
|
548
629
|
options = dict(for_actions=actions, for_resources=resource_names)
|
|
549
630
|
|
|
550
631
|
if ACTIONS.READ in actions:
|
|
551
|
-
config.add_subscriber(
|
|
632
|
+
config.add_subscriber(wrapped_listener, ResourceRead, **options)
|
|
552
633
|
actions = [a for a in actions if a != ACTIONS.READ]
|
|
553
634
|
|
|
554
635
|
if len(actions) > 0:
|
|
555
|
-
config.add_subscriber(
|
|
636
|
+
config.add_subscriber(wrapped_listener, ResourceChanged, **options)
|
|
556
637
|
|
|
557
638
|
|
|
558
639
|
def load_default_settings(config, default_settings):
|
kinto/core/metrics.py
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import types
|
|
2
|
+
|
|
3
|
+
from zope.interface import Interface, implementer
|
|
4
|
+
|
|
5
|
+
from kinto.core import utils
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class IMetricsService(Interface):
|
|
9
|
+
"""
|
|
10
|
+
An interface that defines the metrics service contract.
|
|
11
|
+
Any class implementing this must provide all its methods.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
def timer(key):
|
|
15
|
+
"""
|
|
16
|
+
Watch execution time.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
def observe(self, key, value, labels=[]):
|
|
20
|
+
"""
|
|
21
|
+
Observe a give `value` for the specified `key`.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
def count(key, count=1, unique=None):
|
|
25
|
+
"""
|
|
26
|
+
Count occurrences. If `unique` is set, overwrites the counter value
|
|
27
|
+
on each call.
|
|
28
|
+
|
|
29
|
+
`unique` should be of type ``list[tuple[str,str]]``.
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class NoOpTimer:
|
|
34
|
+
def __call__(self, f):
|
|
35
|
+
@utils.safe_wraps(f)
|
|
36
|
+
def _wrapped(*args, **kwargs):
|
|
37
|
+
return f(*args, **kwargs)
|
|
38
|
+
|
|
39
|
+
return _wrapped
|
|
40
|
+
|
|
41
|
+
def __enter__(self):
|
|
42
|
+
pass
|
|
43
|
+
|
|
44
|
+
def __exit__(self, *args, **kwargs):
|
|
45
|
+
pass
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@implementer(IMetricsService)
|
|
49
|
+
class NoOpMetricsService:
|
|
50
|
+
def timer(self, key):
|
|
51
|
+
return NoOpTimer()
|
|
52
|
+
|
|
53
|
+
def observe(self, key, value, labels=[]):
|
|
54
|
+
pass
|
|
55
|
+
|
|
56
|
+
def count(self, key, count=1, unique=None):
|
|
57
|
+
pass
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def watch_execution_time(metrics_service, obj, prefix="", classname=None):
|
|
61
|
+
"""
|
|
62
|
+
Decorate all methods of an object in order to watch their execution time.
|
|
63
|
+
Metrics will be named `{prefix}.{classname}.{method}`.
|
|
64
|
+
"""
|
|
65
|
+
classname = classname or utils.classname(obj)
|
|
66
|
+
members = dir(obj)
|
|
67
|
+
for name in members:
|
|
68
|
+
value = getattr(obj, name)
|
|
69
|
+
is_method = isinstance(value, types.MethodType)
|
|
70
|
+
if not name.startswith("_") and is_method:
|
|
71
|
+
statsd_key = f"{prefix}.{classname}.{name}"
|
|
72
|
+
decorated_method = metrics_service.timer(statsd_key)(value)
|
|
73
|
+
setattr(obj, name, decorated_method)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def listener_with_timer(config, key, func):
|
|
77
|
+
"""
|
|
78
|
+
Add a timer with the specified `key` on the specified `func`.
|
|
79
|
+
This is used to avoid evaluating `config.registry.metrics` during setup time
|
|
80
|
+
to avoid having to deal with initialization order and configuration committing.
|
|
81
|
+
"""
|
|
82
|
+
|
|
83
|
+
def wrapped(*args, **kwargs):
|
|
84
|
+
metrics_service = config.registry.metrics
|
|
85
|
+
if not metrics_service:
|
|
86
|
+
# This only happens if `kinto.core.initialization.setup_metrics` is
|
|
87
|
+
# not listed in the `initialization_sequence` setting.
|
|
88
|
+
return func(*args, **kwargs)
|
|
89
|
+
# If metrics are enabled, monitor execution time of listeners.
|
|
90
|
+
with metrics_service.timer(key):
|
|
91
|
+
return func(*args, **kwargs)
|
|
92
|
+
|
|
93
|
+
return wrapped
|
|
@@ -246,7 +246,7 @@ class Permission(PermissionBase, MigratorMixin):
|
|
|
246
246
|
|
|
247
247
|
query = f"""
|
|
248
248
|
WITH required_perms AS (
|
|
249
|
-
VALUES {
|
|
249
|
+
VALUES {",".join(perm_values)}
|
|
250
250
|
)
|
|
251
251
|
SELECT principal
|
|
252
252
|
FROM required_perms JOIN access_control_entries
|
|
@@ -292,14 +292,14 @@ class Permission(PermissionBase, MigratorMixin):
|
|
|
292
292
|
object_id_condition = "object_id LIKE pattern"
|
|
293
293
|
else:
|
|
294
294
|
object_id_condition = (
|
|
295
|
-
"object_id LIKE pattern
|
|
295
|
+
"object_id LIKE pattern AND object_id NOT LIKE pattern || '/%'"
|
|
296
296
|
)
|
|
297
297
|
query = f"""
|
|
298
298
|
WITH required_perms AS (
|
|
299
|
-
VALUES {
|
|
299
|
+
VALUES {",".join(perm_values)}
|
|
300
300
|
),
|
|
301
301
|
user_principals AS (
|
|
302
|
-
VALUES {
|
|
302
|
+
VALUES {",".join(principals_values)}
|
|
303
303
|
),
|
|
304
304
|
potential_objects AS (
|
|
305
305
|
SELECT object_id, permission, required_perms.column1 AS pattern
|
|
@@ -341,7 +341,7 @@ class Permission(PermissionBase, MigratorMixin):
|
|
|
341
341
|
|
|
342
342
|
query = f"""
|
|
343
343
|
WITH required_perms AS (
|
|
344
|
-
VALUES {
|
|
344
|
+
VALUES {",".join(perms_values)}
|
|
345
345
|
),
|
|
346
346
|
allowed_principals AS (
|
|
347
347
|
SELECT principal
|
|
@@ -349,7 +349,7 @@ class Permission(PermissionBase, MigratorMixin):
|
|
|
349
349
|
ON (object_id = column1 AND permission = column2)
|
|
350
350
|
),
|
|
351
351
|
required_principals AS (
|
|
352
|
-
VALUES {
|
|
352
|
+
VALUES {",".join(principals_values)}
|
|
353
353
|
)
|
|
354
354
|
SELECT COUNT(*) AS matched
|
|
355
355
|
FROM required_principals JOIN allowed_principals
|
|
@@ -412,7 +412,7 @@ class Permission(PermissionBase, MigratorMixin):
|
|
|
412
412
|
if not new_aces:
|
|
413
413
|
query = f"""
|
|
414
414
|
WITH specified_perms AS (
|
|
415
|
-
VALUES {
|
|
415
|
+
VALUES {",".join(specified_perms)}
|
|
416
416
|
)
|
|
417
417
|
DELETE FROM access_control_entries
|
|
418
418
|
USING specified_perms
|
|
@@ -422,7 +422,7 @@ class Permission(PermissionBase, MigratorMixin):
|
|
|
422
422
|
else:
|
|
423
423
|
query = f"""
|
|
424
424
|
WITH specified_perms AS (
|
|
425
|
-
VALUES {
|
|
425
|
+
VALUES {",".join(specified_perms)}
|
|
426
426
|
),
|
|
427
427
|
delete_specified AS (
|
|
428
428
|
DELETE FROM access_control_entries
|
|
@@ -435,7 +435,7 @@ class Permission(PermissionBase, MigratorMixin):
|
|
|
435
435
|
UNION SELECT :object_id
|
|
436
436
|
),
|
|
437
437
|
new_aces AS (
|
|
438
|
-
VALUES {
|
|
438
|
+
VALUES {",".join(new_aces)}
|
|
439
439
|
)
|
|
440
440
|
INSERT INTO access_control_entries(object_id, permission, principal)
|
|
441
441
|
SELECT DISTINCT d.object_id, n.column1, n.column2
|
kinto/core/resource/__init__.py
CHANGED
|
@@ -665,7 +665,7 @@ class Resource:
|
|
|
665
665
|
obj = self._get_object_or_404(self.object_id)
|
|
666
666
|
self._raise_412_if_modified(obj)
|
|
667
667
|
|
|
668
|
-
#
|
|
668
|
+
# Retrieve the last_modified information from a querystring if present.
|
|
669
669
|
last_modified = self.request.validated["querystring"].get("last_modified")
|
|
670
670
|
|
|
671
671
|
# If less or equal than current object. Ignore it.
|
|
@@ -1058,6 +1058,11 @@ class Resource:
|
|
|
1058
1058
|
|
|
1059
1059
|
def _extract_filters(self):
|
|
1060
1060
|
"""Extracts filters from QueryString parameters."""
|
|
1061
|
+
|
|
1062
|
+
def is_valid_timestamp(value):
|
|
1063
|
+
# Is either integer, or integer as string, or integer between 2 quotes.
|
|
1064
|
+
return isinstance(value, int) or re.match(r'^(\d+)$|^("\d+")$', str(value))
|
|
1065
|
+
|
|
1061
1066
|
queryparams = self.request.validated["querystring"]
|
|
1062
1067
|
|
|
1063
1068
|
filters = []
|
|
@@ -1081,7 +1086,7 @@ class Resource:
|
|
|
1081
1086
|
operator = COMPARISON.GT
|
|
1082
1087
|
else:
|
|
1083
1088
|
if param == "_to":
|
|
1084
|
-
message = "_to is now deprecated,
|
|
1089
|
+
message = "_to is now deprecated, you should use _before instead"
|
|
1085
1090
|
url = (
|
|
1086
1091
|
"https://kinto.readthedocs.io/en/2.4.0/api/"
|
|
1087
1092
|
"resource.html#list-of-available-url-"
|
|
@@ -1090,7 +1095,7 @@ class Resource:
|
|
|
1090
1095
|
send_alert(self.request, message, url)
|
|
1091
1096
|
operator = COMPARISON.LT
|
|
1092
1097
|
|
|
1093
|
-
if value
|
|
1098
|
+
if value is not None and not is_valid_timestamp(value):
|
|
1094
1099
|
raise_invalid(self.request, **error_details)
|
|
1095
1100
|
|
|
1096
1101
|
filters.append(Filter(self.model.modified_field, value, operator))
|
|
@@ -1127,7 +1132,7 @@ class Resource:
|
|
|
1127
1132
|
error_details["description"] = "Invalid character 0x00"
|
|
1128
1133
|
raise_invalid(self.request, **error_details)
|
|
1129
1134
|
|
|
1130
|
-
if field == self.model.modified_field and value
|
|
1135
|
+
if field == self.model.modified_field and not is_valid_timestamp(value):
|
|
1131
1136
|
raise_invalid(self.request, **error_details)
|
|
1132
1137
|
|
|
1133
1138
|
filters.append(Filter(field, value, operator))
|
kinto/core/resource/schema.py
CHANGED
|
@@ -37,8 +37,7 @@ class URL(URL):
|
|
|
37
37
|
|
|
38
38
|
def __init__(self, *args, **kwargs):
|
|
39
39
|
message = (
|
|
40
|
-
"`kinto.core.resource.schema.URL` is deprecated, "
|
|
41
|
-
"use `kinto.core.schema.URL` instead."
|
|
40
|
+
"`kinto.core.resource.schema.URL` is deprecated, use `kinto.core.schema.URL` instead."
|
|
42
41
|
)
|
|
43
42
|
warnings.warn(message, DeprecationWarning)
|
|
44
43
|
super().__init__(*args, **kwargs)
|
kinto/core/statsd.py
CHANGED
|
@@ -1,63 +1 @@
|
|
|
1
|
-
import
|
|
2
|
-
from urllib.parse import urlparse
|
|
3
|
-
|
|
4
|
-
from pyramid.exceptions import ConfigurationError
|
|
5
|
-
|
|
6
|
-
from kinto.core import utils
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
try:
|
|
10
|
-
import statsd as statsd_module
|
|
11
|
-
except ImportError: # pragma: no cover
|
|
12
|
-
statsd_module = None
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
class Client:
|
|
16
|
-
def __init__(self, host, port, prefix):
|
|
17
|
-
self._client = statsd_module.StatsClient(host, port, prefix=prefix)
|
|
18
|
-
|
|
19
|
-
def watch_execution_time(self, obj, prefix="", classname=None):
|
|
20
|
-
classname = classname or utils.classname(obj)
|
|
21
|
-
members = dir(obj)
|
|
22
|
-
for name in members:
|
|
23
|
-
value = getattr(obj, name)
|
|
24
|
-
is_method = isinstance(value, types.MethodType)
|
|
25
|
-
if not name.startswith("_") and is_method:
|
|
26
|
-
statsd_key = f"{prefix}.{classname}.{name}"
|
|
27
|
-
decorated_method = self.timer(statsd_key)(value)
|
|
28
|
-
setattr(obj, name, decorated_method)
|
|
29
|
-
|
|
30
|
-
def timer(self, key):
|
|
31
|
-
return self._client.timer(key)
|
|
32
|
-
|
|
33
|
-
def count(self, key, count=1, unique=None):
|
|
34
|
-
if unique is None:
|
|
35
|
-
return self._client.incr(key, count=count)
|
|
36
|
-
else:
|
|
37
|
-
return self._client.set(key, unique)
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
def statsd_count(request, count_key):
|
|
41
|
-
statsd = request.registry.statsd
|
|
42
|
-
if statsd:
|
|
43
|
-
statsd.count(count_key)
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
def load_from_config(config):
|
|
47
|
-
# If this is called, it means that a ``statsd_url`` was specified in settings.
|
|
48
|
-
# (see ``kinto.core.initialization``)
|
|
49
|
-
# Raise a proper error if the ``statsd`` module is not installed.
|
|
50
|
-
if statsd_module is None:
|
|
51
|
-
error_msg = "Please install Kinto with monitoring dependencies (e.g. statsd package)"
|
|
52
|
-
raise ConfigurationError(error_msg)
|
|
53
|
-
|
|
54
|
-
settings = config.get_settings()
|
|
55
|
-
uri = settings["statsd_url"]
|
|
56
|
-
uri = urlparse(uri)
|
|
57
|
-
|
|
58
|
-
if settings["project_name"] != "":
|
|
59
|
-
prefix = settings["project_name"]
|
|
60
|
-
else:
|
|
61
|
-
prefix = settings["statsd_prefix"]
|
|
62
|
-
|
|
63
|
-
return Client(uri.hostname, uri.port, prefix)
|
|
1
|
+
from kinto.plugins.statsd import load_from_config # noqa: F401
|
|
@@ -95,14 +95,14 @@ def create_from_config(config, prefix="", with_transaction=True):
|
|
|
95
95
|
url = filtered_settings[prefix + "url"]
|
|
96
96
|
existing_client = _CLIENTS[transaction_per_request].get(url)
|
|
97
97
|
if existing_client:
|
|
98
|
-
msg = "Reuse existing PostgreSQL connection.
|
|
98
|
+
msg = f"Reuse existing PostgreSQL connection. Parameters {prefix}* will be ignored."
|
|
99
99
|
warnings.warn(msg)
|
|
100
100
|
return existing_client
|
|
101
101
|
|
|
102
102
|
# Initialize SQLAlchemy engine from filtered_settings.
|
|
103
103
|
poolclass_key = prefix + "poolclass"
|
|
104
104
|
filtered_settings.setdefault(
|
|
105
|
-
poolclass_key, ("kinto.core.storage.postgresql.
|
|
105
|
+
poolclass_key, ("kinto.core.storage.postgresql.pool.QueuePoolWithMaxBacklog")
|
|
106
106
|
)
|
|
107
107
|
filtered_settings[poolclass_key] = config.maybe_dotted(filtered_settings[poolclass_key])
|
|
108
108
|
engine = sqlalchemy.engine_from_config(filtered_settings, prefix=prefix, url=url)
|
kinto/core/testing.py
CHANGED
|
@@ -8,15 +8,19 @@ import webtest
|
|
|
8
8
|
from cornice import errors as cornice_errors
|
|
9
9
|
from pyramid.url import parse_url_overrides
|
|
10
10
|
|
|
11
|
-
from kinto.core import DEFAULT_SETTINGS
|
|
11
|
+
from kinto.core import DEFAULT_SETTINGS
|
|
12
12
|
from kinto.core.storage import generators
|
|
13
13
|
from kinto.core.utils import encode64, follow_subrequest, memcache, sqlalchemy
|
|
14
|
+
from kinto.plugins import prometheus, statsd
|
|
14
15
|
|
|
15
16
|
|
|
16
17
|
skip_if_ci = unittest.skipIf("CI" in os.environ, "ci")
|
|
17
18
|
skip_if_no_postgresql = unittest.skipIf(sqlalchemy is None, "postgresql is not installed.")
|
|
18
19
|
skip_if_no_memcached = unittest.skipIf(memcache is None, "memcached is not installed.")
|
|
19
20
|
skip_if_no_statsd = unittest.skipIf(not statsd.statsd_module, "statsd is not installed.")
|
|
21
|
+
skip_if_no_prometheus = unittest.skipIf(
|
|
22
|
+
not prometheus.prometheus_module, "prometheus is not installed."
|
|
23
|
+
)
|
|
20
24
|
|
|
21
25
|
|
|
22
26
|
class DummyRequest(mock.MagicMock):
|
kinto/core/utils.py
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import collections.abc as collections_abc
|
|
2
|
+
import functools
|
|
2
3
|
import hashlib
|
|
3
4
|
import hmac
|
|
4
5
|
import os
|
|
@@ -261,8 +262,9 @@ def reapply_cors(request, response):
|
|
|
261
262
|
settings = request.registry.settings
|
|
262
263
|
allowed_origins = set(aslist(settings["cors_origins"]))
|
|
263
264
|
required_origins = {"*", origin}
|
|
264
|
-
|
|
265
|
-
|
|
265
|
+
matches = allowed_origins.intersection(required_origins)
|
|
266
|
+
if matches:
|
|
267
|
+
response.headers["Access-Control-Allow-Origin"] = matches.pop()
|
|
266
268
|
|
|
267
269
|
# Import service here because kinto.core import utils
|
|
268
270
|
from kinto.core import Service
|
|
@@ -541,3 +543,10 @@ def apply_json_patch(obj, ops):
|
|
|
541
543
|
raise ValueError(e)
|
|
542
544
|
|
|
543
545
|
return result
|
|
546
|
+
|
|
547
|
+
|
|
548
|
+
def safe_wraps(wrapper, *args, **kwargs):
|
|
549
|
+
"""Safely wraps partial functions."""
|
|
550
|
+
while isinstance(wrapper, functools.partial):
|
|
551
|
+
wrapper = wrapper.func
|
|
552
|
+
return functools.wraps(wrapper, *args, **kwargs)
|
kinto/core/views/errors.py
CHANGED
|
@@ -53,7 +53,7 @@ def page_not_found(response, request):
|
|
|
53
53
|
|
|
54
54
|
if not request.path.startswith(f"/{request.registry.route_prefix}"):
|
|
55
55
|
errno = ERRORS.VERSION_NOT_AVAILABLE
|
|
56
|
-
error_msg = "The requested API version is not available
|
|
56
|
+
error_msg = "The requested API version is not available on this server."
|
|
57
57
|
elif trailing_slash_redirection_enabled:
|
|
58
58
|
redirect = None
|
|
59
59
|
|
|
@@ -80,8 +80,7 @@ def page_not_found(response, request):
|
|
|
80
80
|
def service_unavailable(response, request):
|
|
81
81
|
if response.content_type != "application/json":
|
|
82
82
|
error_msg = (
|
|
83
|
-
"Service temporary unavailable "
|
|
84
|
-
"due to overloading or maintenance, please retry later."
|
|
83
|
+
"Service temporary unavailable due to overloading or maintenance, please retry later."
|
|
85
84
|
)
|
|
86
85
|
response = http_error(response, errno=ERRORS.BACKEND, message=error_msg)
|
|
87
86
|
|
|
@@ -83,8 +83,7 @@ def includeme(config):
|
|
|
83
83
|
if "basicauth" in auth_policies and policy in auth_policies:
|
|
84
84
|
if auth_policies.index("basicauth") < auth_policies.index(policy):
|
|
85
85
|
error_msg = (
|
|
86
|
-
"'basicauth' should not be mentioned before '%s' "
|
|
87
|
-
"in 'multiauth.policies' setting."
|
|
86
|
+
"'basicauth' should not be mentioned before '%s' in 'multiauth.policies' setting."
|
|
88
87
|
) % policy
|
|
89
88
|
raise ConfigurationError(error_msg)
|
|
90
89
|
|