kinto 18.1.1__py3-none-any.whl → 19.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of kinto might be problematic. Click here for more details.
- kinto/core/__init__.py +1 -1
- kinto/core/initialization.py +82 -53
- kinto/core/metrics.py +57 -0
- kinto/core/statsd.py +1 -63
- kinto/core/testing.py +5 -1
- kinto/plugins/history/__init__.py +5 -6
- kinto/plugins/prometheus.py +150 -0
- kinto/plugins/quotas/__init__.py +5 -6
- kinto/plugins/statsd.py +58 -0
- {kinto-18.1.1.dist-info → kinto-19.2.0.dist-info}/METADATA +3 -2
- {kinto-18.1.1.dist-info → kinto-19.2.0.dist-info}/RECORD +15 -12
- {kinto-18.1.1.dist-info → kinto-19.2.0.dist-info}/WHEEL +1 -1
- {kinto-18.1.1.dist-info → kinto-19.2.0.dist-info}/LICENSE +0 -0
- {kinto-18.1.1.dist-info → kinto-19.2.0.dist-info}/entry_points.txt +0 -0
- {kinto-18.1.1.dist-info → kinto-19.2.0.dist-info}/top_level.txt +0 -0
kinto/core/__init__.py
CHANGED
|
@@ -66,8 +66,8 @@ DEFAULT_SETTINGS = {
|
|
|
66
66
|
"kinto.core.initialization.setup_authentication",
|
|
67
67
|
"kinto.core.initialization.setup_backoff",
|
|
68
68
|
"kinto.core.initialization.setup_sentry",
|
|
69
|
-
"kinto.core.initialization.setup_statsd",
|
|
70
69
|
"kinto.core.initialization.setup_listeners",
|
|
70
|
+
"kinto.core.initialization.setup_metrics",
|
|
71
71
|
"kinto.core.events.setup_transaction_hook",
|
|
72
72
|
),
|
|
73
73
|
"event_listeners": "",
|
kinto/core/initialization.py
CHANGED
|
@@ -21,7 +21,7 @@ from pyramid.security import NO_PERMISSION_REQUIRED
|
|
|
21
21
|
from pyramid.settings import asbool, aslist
|
|
22
22
|
from pyramid_multiauth import MultiAuthenticationPolicy, MultiAuthPolicySelected
|
|
23
23
|
|
|
24
|
-
from kinto.core import cache, errors, permission, storage, utils
|
|
24
|
+
from kinto.core import cache, errors, metrics, permission, storage, utils
|
|
25
25
|
from kinto.core.events import ACTIONS, ResourceChanged, ResourceRead
|
|
26
26
|
|
|
27
27
|
|
|
@@ -334,51 +334,13 @@ def setup_sentry(config):
|
|
|
334
334
|
|
|
335
335
|
|
|
336
336
|
def setup_statsd(config):
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
config.registry.statsd = client
|
|
346
|
-
|
|
347
|
-
client.watch_execution_time(config.registry.cache, prefix="backend")
|
|
348
|
-
client.watch_execution_time(config.registry.storage, prefix="backend")
|
|
349
|
-
client.watch_execution_time(config.registry.permission, prefix="backend")
|
|
350
|
-
|
|
351
|
-
# Commit so that configured policy can be queried.
|
|
352
|
-
config.commit()
|
|
353
|
-
policy = config.registry.queryUtility(IAuthenticationPolicy)
|
|
354
|
-
if isinstance(policy, MultiAuthenticationPolicy):
|
|
355
|
-
for name, subpolicy in policy.get_policies():
|
|
356
|
-
client.watch_execution_time(subpolicy, prefix="authentication", classname=name)
|
|
357
|
-
else:
|
|
358
|
-
client.watch_execution_time(policy, prefix="authentication")
|
|
359
|
-
|
|
360
|
-
def on_new_response(event):
|
|
361
|
-
request = event.request
|
|
362
|
-
|
|
363
|
-
# Count unique users.
|
|
364
|
-
user_id = request.prefixed_userid
|
|
365
|
-
if user_id:
|
|
366
|
-
# Get rid of colons in metric packet (see #1282).
|
|
367
|
-
user_id = user_id.replace(":", ".")
|
|
368
|
-
client.count("users", unique=user_id)
|
|
369
|
-
|
|
370
|
-
# Count authentication verifications.
|
|
371
|
-
if hasattr(request, "authn_type"):
|
|
372
|
-
client.count(f"authn_type.{request.authn_type}")
|
|
373
|
-
|
|
374
|
-
# Count view calls.
|
|
375
|
-
service = request.current_service
|
|
376
|
-
if service:
|
|
377
|
-
client.count(f"view.{service.name}.{request.method}")
|
|
378
|
-
|
|
379
|
-
config.add_subscriber(on_new_response, NewResponse)
|
|
380
|
-
|
|
381
|
-
return client
|
|
337
|
+
# It would be pretty rare to find users that have a custom ``kinto.initialization_sequence`` setting.
|
|
338
|
+
# But just in case, warn that it will be removed in next major.
|
|
339
|
+
warnings.warn(
|
|
340
|
+
"``setup_statsd()`` is now deprecated. Use ``kinto.core.initialization.setup_metrics()`` instead.",
|
|
341
|
+
DeprecationWarning,
|
|
342
|
+
)
|
|
343
|
+
setup_metrics(config)
|
|
382
344
|
|
|
383
345
|
|
|
384
346
|
def install_middlewares(app, settings):
|
|
@@ -466,6 +428,75 @@ def setup_logging(config):
|
|
|
466
428
|
config.add_subscriber(on_new_response, NewResponse)
|
|
467
429
|
|
|
468
430
|
|
|
431
|
+
def setup_metrics(config):
|
|
432
|
+
settings = config.get_settings()
|
|
433
|
+
|
|
434
|
+
# This does not fully respect the Pyramid/ZCA patterns, but the rest of Kinto uses
|
|
435
|
+
# `registry.storage`, `registry.cache`, etc. Consistency seems more important.
|
|
436
|
+
config.registry.__class__.metrics = property(
|
|
437
|
+
lambda reg: reg.queryUtility(metrics.IMetricsService)
|
|
438
|
+
)
|
|
439
|
+
|
|
440
|
+
def deprecated_registry(self):
|
|
441
|
+
warnings.warn(
|
|
442
|
+
"``config.registry.statsd`` is now deprecated. Use ``config.registry.metrics`` instead.",
|
|
443
|
+
DeprecationWarning,
|
|
444
|
+
)
|
|
445
|
+
return self.metrics
|
|
446
|
+
|
|
447
|
+
config.registry.__class__.statsd = property(deprecated_registry)
|
|
448
|
+
|
|
449
|
+
def on_app_created(event):
|
|
450
|
+
config = event.app
|
|
451
|
+
metrics_service = config.registry.metrics
|
|
452
|
+
if not metrics_service:
|
|
453
|
+
logger.warning("No metrics service registered.")
|
|
454
|
+
return
|
|
455
|
+
|
|
456
|
+
metrics.watch_execution_time(metrics_service, config.registry.cache, prefix="backend")
|
|
457
|
+
metrics.watch_execution_time(metrics_service, config.registry.storage, prefix="backend")
|
|
458
|
+
metrics.watch_execution_time(metrics_service, config.registry.permission, prefix="backend")
|
|
459
|
+
|
|
460
|
+
policy = config.registry.queryUtility(IAuthenticationPolicy)
|
|
461
|
+
if isinstance(policy, MultiAuthenticationPolicy):
|
|
462
|
+
for name, subpolicy in policy.get_policies():
|
|
463
|
+
metrics.watch_execution_time(
|
|
464
|
+
metrics_service, subpolicy, prefix="authentication", classname=name
|
|
465
|
+
)
|
|
466
|
+
else:
|
|
467
|
+
metrics.watch_execution_time(metrics_service, policy, prefix="authentication")
|
|
468
|
+
|
|
469
|
+
config.add_subscriber(on_app_created, ApplicationCreated)
|
|
470
|
+
|
|
471
|
+
def on_new_response(event):
|
|
472
|
+
request = event.request
|
|
473
|
+
metrics_service = config.registry.metrics
|
|
474
|
+
if not metrics_service:
|
|
475
|
+
return
|
|
476
|
+
|
|
477
|
+
# Count unique users.
|
|
478
|
+
user_id = request.prefixed_userid
|
|
479
|
+
if user_id:
|
|
480
|
+
# Get rid of colons in metric packet (see #1282).
|
|
481
|
+
user_id = user_id.replace(":", ".")
|
|
482
|
+
metrics_service.count("users", unique=user_id)
|
|
483
|
+
|
|
484
|
+
# Count authentication verifications.
|
|
485
|
+
if hasattr(request, "authn_type"):
|
|
486
|
+
metrics_service.count(f"authn_type.{request.authn_type}")
|
|
487
|
+
|
|
488
|
+
# Count view calls.
|
|
489
|
+
service = request.current_service
|
|
490
|
+
if service:
|
|
491
|
+
metrics_service.count(f"view.{service.name}.{request.method}")
|
|
492
|
+
|
|
493
|
+
config.add_subscriber(on_new_response, NewResponse)
|
|
494
|
+
|
|
495
|
+
# While statsd is deprecated, we include its plugin by default for retro-compability.
|
|
496
|
+
if settings["statsd_url"]:
|
|
497
|
+
config.include("kinto.plugins.statsd")
|
|
498
|
+
|
|
499
|
+
|
|
469
500
|
class EventActionFilter:
|
|
470
501
|
def __init__(self, actions, config):
|
|
471
502
|
actions = ACTIONS.from_string_list(actions)
|
|
@@ -518,11 +549,9 @@ def setup_listeners(config):
|
|
|
518
549
|
listener_mod = config.maybe_dotted(module_value)
|
|
519
550
|
listener = listener_mod.load_from_config(config, prefix)
|
|
520
551
|
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
key = f"listeners.{name}"
|
|
525
|
-
listener = statsd_client.timer(key)(listener.__call__)
|
|
552
|
+
wrapped_listener = metrics.listener_with_timer(
|
|
553
|
+
config, f"listeners.{name}", listener.__call__
|
|
554
|
+
)
|
|
526
555
|
|
|
527
556
|
# Optional filter by event action.
|
|
528
557
|
actions_setting = prefix + "actions"
|
|
@@ -548,11 +577,11 @@ def setup_listeners(config):
|
|
|
548
577
|
options = dict(for_actions=actions, for_resources=resource_names)
|
|
549
578
|
|
|
550
579
|
if ACTIONS.READ in actions:
|
|
551
|
-
config.add_subscriber(
|
|
580
|
+
config.add_subscriber(wrapped_listener, ResourceRead, **options)
|
|
552
581
|
actions = [a for a in actions if a != ACTIONS.READ]
|
|
553
582
|
|
|
554
583
|
if len(actions) > 0:
|
|
555
|
-
config.add_subscriber(
|
|
584
|
+
config.add_subscriber(wrapped_listener, ResourceChanged, **options)
|
|
556
585
|
|
|
557
586
|
|
|
558
587
|
def load_default_settings(config, default_settings):
|
kinto/core/metrics.py
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import types
|
|
2
|
+
|
|
3
|
+
from zope.interface import Interface
|
|
4
|
+
|
|
5
|
+
from kinto.core import utils
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class IMetricsService(Interface):
|
|
9
|
+
"""
|
|
10
|
+
An interface that defines the metrics service contract.
|
|
11
|
+
Any class implementing this must provide all its methods.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
def timer(key):
|
|
15
|
+
"""
|
|
16
|
+
Watch execution time.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
def count(key, count=1, unique=None):
|
|
20
|
+
"""
|
|
21
|
+
Count occurrences. If `unique` is set, overwrites the counter value
|
|
22
|
+
on each call.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def watch_execution_time(metrics_service, obj, prefix="", classname=None):
|
|
27
|
+
"""
|
|
28
|
+
Decorate all methods of an object in order to watch their execution time.
|
|
29
|
+
Metrics will be named `{prefix}.{classname}.{method}`.
|
|
30
|
+
"""
|
|
31
|
+
classname = classname or utils.classname(obj)
|
|
32
|
+
members = dir(obj)
|
|
33
|
+
for name in members:
|
|
34
|
+
value = getattr(obj, name)
|
|
35
|
+
is_method = isinstance(value, types.MethodType)
|
|
36
|
+
if not name.startswith("_") and is_method:
|
|
37
|
+
statsd_key = f"{prefix}.{classname}.{name}"
|
|
38
|
+
decorated_method = metrics_service.timer(statsd_key)(value)
|
|
39
|
+
setattr(obj, name, decorated_method)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def listener_with_timer(config, key, func):
|
|
43
|
+
"""
|
|
44
|
+
Add a timer with the specified `key` on the specified `func`.
|
|
45
|
+
This is used to avoid evaluating `config.registry.metrics` during setup time
|
|
46
|
+
to avoid having to deal with initialization order and configuration committing.
|
|
47
|
+
"""
|
|
48
|
+
|
|
49
|
+
def wrapped(*args, **kwargs):
|
|
50
|
+
metrics_service = config.registry.metrics
|
|
51
|
+
if not metrics_service:
|
|
52
|
+
return func(*args, **kwargs)
|
|
53
|
+
# If metrics are enabled, monitor execution time of listeners.
|
|
54
|
+
with metrics_service.timer(key):
|
|
55
|
+
return func(*args, **kwargs)
|
|
56
|
+
|
|
57
|
+
return wrapped
|
kinto/core/statsd.py
CHANGED
|
@@ -1,63 +1 @@
|
|
|
1
|
-
import
|
|
2
|
-
from urllib.parse import urlparse
|
|
3
|
-
|
|
4
|
-
from pyramid.exceptions import ConfigurationError
|
|
5
|
-
|
|
6
|
-
from kinto.core import utils
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
try:
|
|
10
|
-
import statsd as statsd_module
|
|
11
|
-
except ImportError: # pragma: no cover
|
|
12
|
-
statsd_module = None
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
class Client:
|
|
16
|
-
def __init__(self, host, port, prefix):
|
|
17
|
-
self._client = statsd_module.StatsClient(host, port, prefix=prefix)
|
|
18
|
-
|
|
19
|
-
def watch_execution_time(self, obj, prefix="", classname=None):
|
|
20
|
-
classname = classname or utils.classname(obj)
|
|
21
|
-
members = dir(obj)
|
|
22
|
-
for name in members:
|
|
23
|
-
value = getattr(obj, name)
|
|
24
|
-
is_method = isinstance(value, types.MethodType)
|
|
25
|
-
if not name.startswith("_") and is_method:
|
|
26
|
-
statsd_key = f"{prefix}.{classname}.{name}"
|
|
27
|
-
decorated_method = self.timer(statsd_key)(value)
|
|
28
|
-
setattr(obj, name, decorated_method)
|
|
29
|
-
|
|
30
|
-
def timer(self, key):
|
|
31
|
-
return self._client.timer(key)
|
|
32
|
-
|
|
33
|
-
def count(self, key, count=1, unique=None):
|
|
34
|
-
if unique is None:
|
|
35
|
-
return self._client.incr(key, count=count)
|
|
36
|
-
else:
|
|
37
|
-
return self._client.set(key, unique)
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
def statsd_count(request, count_key):
|
|
41
|
-
statsd = request.registry.statsd
|
|
42
|
-
if statsd:
|
|
43
|
-
statsd.count(count_key)
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
def load_from_config(config):
|
|
47
|
-
# If this is called, it means that a ``statsd_url`` was specified in settings.
|
|
48
|
-
# (see ``kinto.core.initialization``)
|
|
49
|
-
# Raise a proper error if the ``statsd`` module is not installed.
|
|
50
|
-
if statsd_module is None:
|
|
51
|
-
error_msg = "Please install Kinto with monitoring dependencies (e.g. statsd package)"
|
|
52
|
-
raise ConfigurationError(error_msg)
|
|
53
|
-
|
|
54
|
-
settings = config.get_settings()
|
|
55
|
-
uri = settings["statsd_url"]
|
|
56
|
-
uri = urlparse(uri)
|
|
57
|
-
|
|
58
|
-
if settings["project_name"] != "":
|
|
59
|
-
prefix = settings["project_name"]
|
|
60
|
-
else:
|
|
61
|
-
prefix = settings["statsd_prefix"]
|
|
62
|
-
|
|
63
|
-
return Client(uri.hostname, uri.port, prefix)
|
|
1
|
+
from kinto.plugins.statsd import load_from_config # noqa: F401
|
kinto/core/testing.py
CHANGED
|
@@ -8,15 +8,19 @@ import webtest
|
|
|
8
8
|
from cornice import errors as cornice_errors
|
|
9
9
|
from pyramid.url import parse_url_overrides
|
|
10
10
|
|
|
11
|
-
from kinto.core import DEFAULT_SETTINGS
|
|
11
|
+
from kinto.core import DEFAULT_SETTINGS
|
|
12
12
|
from kinto.core.storage import generators
|
|
13
13
|
from kinto.core.utils import encode64, follow_subrequest, memcache, sqlalchemy
|
|
14
|
+
from kinto.plugins import prometheus, statsd
|
|
14
15
|
|
|
15
16
|
|
|
16
17
|
skip_if_ci = unittest.skipIf("CI" in os.environ, "ci")
|
|
17
18
|
skip_if_no_postgresql = unittest.skipIf(sqlalchemy is None, "postgresql is not installed.")
|
|
18
19
|
skip_if_no_memcached = unittest.skipIf(memcache is None, "memcached is not installed.")
|
|
19
20
|
skip_if_no_statsd = unittest.skipIf(not statsd.statsd_module, "statsd is not installed.")
|
|
21
|
+
skip_if_no_prometheus = unittest.skipIf(
|
|
22
|
+
not prometheus.prometheus_module, "prometheus is not installed."
|
|
23
|
+
)
|
|
20
24
|
|
|
21
25
|
|
|
22
26
|
class DummyRequest(mock.MagicMock):
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
from kinto.authorization import PERMISSIONS_INHERITANCE_TREE
|
|
2
|
+
from kinto.core import metrics
|
|
2
3
|
from kinto.core.events import ResourceChanged
|
|
3
4
|
|
|
4
5
|
from .listener import on_resource_changed
|
|
@@ -14,15 +15,13 @@ def includeme(config):
|
|
|
14
15
|
# Activate end-points.
|
|
15
16
|
config.scan("kinto.plugins.history.views")
|
|
16
17
|
|
|
17
|
-
|
|
18
|
-
listener = on_resource_changed
|
|
19
|
-
if config.registry.statsd:
|
|
20
|
-
key = "plugins.history"
|
|
21
|
-
listener = config.registry.statsd.timer(key)(on_resource_changed)
|
|
18
|
+
wrapped_listener = metrics.listener_with_timer(config, "plugins.history", on_resource_changed)
|
|
22
19
|
|
|
23
20
|
# Listen to every resources (except history)
|
|
24
21
|
config.add_subscriber(
|
|
25
|
-
|
|
22
|
+
wrapped_listener,
|
|
23
|
+
ResourceChanged,
|
|
24
|
+
for_resources=("bucket", "group", "collection", "record"),
|
|
26
25
|
)
|
|
27
26
|
|
|
28
27
|
# Register the permission inheritance for history entries.
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
import functools
|
|
2
|
+
from time import perf_counter as time_now
|
|
3
|
+
|
|
4
|
+
from pyramid.exceptions import ConfigurationError
|
|
5
|
+
from pyramid.response import Response
|
|
6
|
+
from zope.interface import implementer
|
|
7
|
+
|
|
8
|
+
from kinto.core import metrics
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
try:
|
|
12
|
+
import prometheus_client as prometheus_module
|
|
13
|
+
except ImportError: # pragma: no cover
|
|
14
|
+
prometheus_module = None
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
_METRICS = {}
|
|
18
|
+
_REGISTRY = None
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def get_registry():
|
|
22
|
+
global _REGISTRY
|
|
23
|
+
|
|
24
|
+
if _REGISTRY is None:
|
|
25
|
+
_REGISTRY = prometheus_module.CollectorRegistry()
|
|
26
|
+
return _REGISTRY
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _fix_metric_name(s):
|
|
30
|
+
return s.replace("-", "_").replace(".", "_")
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def safe_wraps(wrapper, *args, **kwargs):
|
|
34
|
+
"""Safely wraps partial functions."""
|
|
35
|
+
while isinstance(wrapper, functools.partial):
|
|
36
|
+
wrapper = wrapper.func
|
|
37
|
+
return functools.wraps(wrapper, *args, **kwargs)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class Timer:
|
|
41
|
+
def __init__(self, summary):
|
|
42
|
+
self.summary = summary
|
|
43
|
+
self._start_time = None
|
|
44
|
+
|
|
45
|
+
def __call__(self, f):
|
|
46
|
+
@safe_wraps(f)
|
|
47
|
+
def _wrapped(*args, **kwargs):
|
|
48
|
+
start_time = time_now()
|
|
49
|
+
try:
|
|
50
|
+
return f(*args, **kwargs)
|
|
51
|
+
finally:
|
|
52
|
+
dt_ms = 1000.0 * (time_now() - start_time)
|
|
53
|
+
self.summary.observe(dt_ms)
|
|
54
|
+
|
|
55
|
+
return _wrapped
|
|
56
|
+
|
|
57
|
+
def __enter__(self):
|
|
58
|
+
return self.start()
|
|
59
|
+
|
|
60
|
+
def __exit__(self, typ, value, tb):
|
|
61
|
+
self.stop()
|
|
62
|
+
|
|
63
|
+
def start(self):
|
|
64
|
+
self._start_time = time_now()
|
|
65
|
+
return self
|
|
66
|
+
|
|
67
|
+
def stop(self):
|
|
68
|
+
if self._start_time is None: # pragma: nocover
|
|
69
|
+
raise RuntimeError("Timer has not started.")
|
|
70
|
+
dt_ms = 1000.0 * (time_now() - self._start_time)
|
|
71
|
+
self.summary.observe(dt_ms)
|
|
72
|
+
return self
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
@implementer(metrics.IMetricsService)
|
|
76
|
+
class PrometheusService:
|
|
77
|
+
def timer(self, key):
|
|
78
|
+
global _METRICS
|
|
79
|
+
if key not in _METRICS:
|
|
80
|
+
_METRICS[key] = prometheus_module.Summary(
|
|
81
|
+
_fix_metric_name(key), f"Summary of {key}", registry=get_registry()
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
if not isinstance(_METRICS[key], prometheus_module.Summary):
|
|
85
|
+
raise RuntimeError(
|
|
86
|
+
f"Metric {key} already exists with different type ({_METRICS[key]})"
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
return Timer(_METRICS[key])
|
|
90
|
+
|
|
91
|
+
def count(self, key, count=1, unique=None):
|
|
92
|
+
global _METRICS
|
|
93
|
+
|
|
94
|
+
# Turn `unique` into a group and a value:
|
|
95
|
+
# eg. `method.basicauth.mat` -> `method_basicauth="mat"`
|
|
96
|
+
label_value = None
|
|
97
|
+
if unique:
|
|
98
|
+
if "." not in unique:
|
|
99
|
+
unique = f"group.{unique}"
|
|
100
|
+
label_name, label_value = unique.rsplit(".", 1)
|
|
101
|
+
label_names = (_fix_metric_name(label_name),)
|
|
102
|
+
else:
|
|
103
|
+
label_names = tuple()
|
|
104
|
+
|
|
105
|
+
if key not in _METRICS:
|
|
106
|
+
_METRICS[key] = prometheus_module.Counter(
|
|
107
|
+
_fix_metric_name(key),
|
|
108
|
+
f"Counter of {key}",
|
|
109
|
+
labelnames=label_names,
|
|
110
|
+
registry=get_registry(),
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
if not isinstance(_METRICS[key], prometheus_module.Counter):
|
|
114
|
+
raise RuntimeError(
|
|
115
|
+
f"Metric {key} already exists with different type ({_METRICS[key]})"
|
|
116
|
+
)
|
|
117
|
+
|
|
118
|
+
m = _METRICS[key]
|
|
119
|
+
if label_value is not None:
|
|
120
|
+
m = m.labels(label_value)
|
|
121
|
+
|
|
122
|
+
m.inc(count)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def metrics_view(request):
|
|
126
|
+
registry = get_registry()
|
|
127
|
+
data = prometheus_module.generate_latest(registry)
|
|
128
|
+
resp = Response(body=data)
|
|
129
|
+
resp.headers["Content-Type"] = prometheus_module.CONTENT_TYPE_LATEST
|
|
130
|
+
resp.headers["Content-Length"] = str(len(data))
|
|
131
|
+
return resp
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def includeme(config):
|
|
135
|
+
if prometheus_module is None:
|
|
136
|
+
error_msg = (
|
|
137
|
+
"Please install Kinto with monitoring dependencies (e.g. prometheus-client package)"
|
|
138
|
+
)
|
|
139
|
+
raise ConfigurationError(error_msg)
|
|
140
|
+
|
|
141
|
+
config.add_api_capability(
|
|
142
|
+
"prometheus",
|
|
143
|
+
description="Prometheus metrics.",
|
|
144
|
+
url="https://github.com/Kinto/kinto/",
|
|
145
|
+
)
|
|
146
|
+
|
|
147
|
+
config.add_route("prometheus_metrics", "/__metrics__")
|
|
148
|
+
config.add_view(metrics_view, route_name="prometheus_metrics")
|
|
149
|
+
|
|
150
|
+
config.registry.registerUtility(PrometheusService(), metrics.IMetricsService)
|
kinto/plugins/quotas/__init__.py
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
from kinto.core import metrics
|
|
1
2
|
from kinto.core.events import ResourceChanged
|
|
2
3
|
|
|
3
4
|
from .listener import on_resource_changed
|
|
@@ -10,13 +11,11 @@ def includeme(config):
|
|
|
10
11
|
url="https://kinto.readthedocs.io",
|
|
11
12
|
)
|
|
12
13
|
|
|
13
|
-
|
|
14
|
-
listener = on_resource_changed
|
|
15
|
-
if config.registry.statsd:
|
|
16
|
-
key = "plugins.quotas"
|
|
17
|
-
listener = config.registry.statsd.timer(key)(on_resource_changed)
|
|
14
|
+
wrapped_listener = metrics.listener_with_timer(config, "plugins.quotas", on_resource_changed)
|
|
18
15
|
|
|
19
16
|
# Listen to every resources (except history)
|
|
20
17
|
config.add_subscriber(
|
|
21
|
-
|
|
18
|
+
wrapped_listener,
|
|
19
|
+
ResourceChanged,
|
|
20
|
+
for_resources=("bucket", "group", "collection", "record"),
|
|
22
21
|
)
|
kinto/plugins/statsd.py
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
from urllib.parse import urlparse
|
|
2
|
+
|
|
3
|
+
from pyramid.exceptions import ConfigurationError
|
|
4
|
+
from zope.interface import implementer
|
|
5
|
+
|
|
6
|
+
from kinto.core import metrics
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
try:
|
|
10
|
+
import statsd as statsd_module
|
|
11
|
+
except ImportError: # pragma: no cover
|
|
12
|
+
statsd_module = None
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@implementer(metrics.IMetricsService)
|
|
16
|
+
class StatsDService:
|
|
17
|
+
def __init__(self, host, port, prefix):
|
|
18
|
+
self._client = statsd_module.StatsClient(host, port, prefix=prefix)
|
|
19
|
+
|
|
20
|
+
def timer(self, key):
|
|
21
|
+
return self._client.timer(key)
|
|
22
|
+
|
|
23
|
+
def count(self, key, count=1, unique=None):
|
|
24
|
+
if unique is None:
|
|
25
|
+
return self._client.incr(key, count=count)
|
|
26
|
+
else:
|
|
27
|
+
return self._client.set(key, unique)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def load_from_config(config):
|
|
31
|
+
# If this is called, it means that a ``statsd_url`` was specified in settings.
|
|
32
|
+
# (see ``kinto.core.initialization``)
|
|
33
|
+
# Raise a proper error if the ``statsd`` module is not installed.
|
|
34
|
+
if statsd_module is None:
|
|
35
|
+
error_msg = "Please install Kinto with monitoring dependencies (e.g. statsd package)"
|
|
36
|
+
raise ConfigurationError(error_msg)
|
|
37
|
+
|
|
38
|
+
settings = config.get_settings()
|
|
39
|
+
uri = settings["statsd_url"]
|
|
40
|
+
uri = urlparse(uri)
|
|
41
|
+
|
|
42
|
+
if settings["project_name"] != "":
|
|
43
|
+
prefix = settings["project_name"]
|
|
44
|
+
else:
|
|
45
|
+
prefix = settings["statsd_prefix"]
|
|
46
|
+
|
|
47
|
+
return StatsDService(uri.hostname, uri.port, prefix)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def includeme(config):
|
|
51
|
+
settings = config.get_settings()
|
|
52
|
+
|
|
53
|
+
# TODO: this backend abstraction may not be required anymore.
|
|
54
|
+
statsd_mod = settings["statsd_backend"]
|
|
55
|
+
statsd_mod = config.maybe_dotted(statsd_mod)
|
|
56
|
+
client = statsd_mod.load_from_config(config)
|
|
57
|
+
|
|
58
|
+
config.registry.registerUtility(client, metrics.IMetricsService)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: kinto
|
|
3
|
-
Version:
|
|
3
|
+
Version: 19.2.0
|
|
4
4
|
Summary: Kinto Web Service - Store, Sync, Share, and Self-Host.
|
|
5
5
|
Author-email: Mozilla Services <developers@kinto-storage.org>
|
|
6
6
|
License: Copyright 2012 - Mozilla Foundation
|
|
@@ -59,6 +59,7 @@ Requires-Dist: newrelic ; extra == 'monitoring'
|
|
|
59
59
|
Requires-Dist: sentry-sdk[sqlalchemy] ; extra == 'monitoring'
|
|
60
60
|
Requires-Dist: statsd ; extra == 'monitoring'
|
|
61
61
|
Requires-Dist: werkzeug ; extra == 'monitoring'
|
|
62
|
+
Requires-Dist: prometheus-client ; extra == 'monitoring'
|
|
62
63
|
Provides-Extra: postgresql
|
|
63
64
|
Requires-Dist: SQLAlchemy <3 ; extra == 'postgresql'
|
|
64
65
|
Requires-Dist: psycopg2 ; extra == 'postgresql'
|
|
@@ -79,7 +80,7 @@ Kinto
|
|
|
79
80
|
|coc| |gitter| |readthedocs| |pypi| |ci| |main-coverage|
|
|
80
81
|
|
|
81
82
|
.. |coc| image:: https://img.shields.io/badge/%E2%9D%A4-code%20of%20conduct-blue.svg
|
|
82
|
-
:target: https://github.com/Kinto/kinto/blob/main/CODE_OF_CONDUCT.md
|
|
83
|
+
:target: https://github.com/Kinto/kinto/blob/main/.github/CODE_OF_CONDUCT.md
|
|
83
84
|
:alt: Code of conduct
|
|
84
85
|
|
|
85
86
|
.. |gitter| image:: https://badges.gitter.im/Kinto/kinto.svg
|
|
@@ -6,18 +6,19 @@ kinto/schema_validation.py,sha256=mtAmnl5HwiUsjS2gU8MKH4lkZ1380A5wZht-w9s5X7M,53
|
|
|
6
6
|
kinto/scripts.py,sha256=oM8ggBofYVoVhxrbdfzwIO0AOwR6Z2NJ2RqH9op1_lg,1370
|
|
7
7
|
kinto/config/__init__.py,sha256=av8W0utmjheueFqrjTYEDk_vbpm3XYdHcqv5lppNR4k,2131
|
|
8
8
|
kinto/config/kinto.tpl,sha256=kdA2RuR8gEpZGJp6VkKqMShQ0CfHDM0gjUttSPeaT7s,8753
|
|
9
|
-
kinto/core/__init__.py,sha256=
|
|
9
|
+
kinto/core/__init__.py,sha256=vaCYHKF-OjmBsl-Q4jgA_BbnUn_cvfH3p6-UDe03qIM,8126
|
|
10
10
|
kinto/core/authentication.py,sha256=HLA0kREC3GMEsrIsHsQYjVNztYfAF01kb8-pLboByFs,1527
|
|
11
11
|
kinto/core/authorization.py,sha256=GywY25KEzuSSAI709dFHDfdLnKxy3SLEYGwW5FkQ7Qc,13212
|
|
12
12
|
kinto/core/decorators.py,sha256=3SAPWXlyPNUSICZ9mz04bcN-UdbnDuFOtU0bQHHzLis,2178
|
|
13
13
|
kinto/core/errors.py,sha256=fxOLR4lImwHp26hhxxaqk6uW0U7v2Jb6f4sgAcRPRu8,8854
|
|
14
14
|
kinto/core/events.py,sha256=SYpXgKMtVjiD9fwYJA2Omdom9yA3nBqi9btdvU1I_nc,10345
|
|
15
|
-
kinto/core/initialization.py,sha256=
|
|
15
|
+
kinto/core/initialization.py,sha256=Z4yNDw0qzernxSzMI7t_KkONblh9oe_01r_8GeS0XNI,24572
|
|
16
|
+
kinto/core/metrics.py,sha256=IWNh1p46sNUWJJu1lJgILoyUTXRw1eM3KRjsvkyOh88,1766
|
|
16
17
|
kinto/core/openapi.py,sha256=bPHRCVmdZsJCsKm8BId02Q2Wz8etF1xL7Z9rU9JDx5k,3855
|
|
17
18
|
kinto/core/schema.py,sha256=d5L5TQynRYJPkZ8Mu2X7F72xEh6SKDbrHK1CNTdOf2E,3646
|
|
18
19
|
kinto/core/scripts.py,sha256=5HSq5QAuin7HuU6icNYkPisny-4JpcdBvjf8X4JImrE,978
|
|
19
|
-
kinto/core/statsd.py,sha256=
|
|
20
|
-
kinto/core/testing.py,sha256=
|
|
20
|
+
kinto/core/statsd.py,sha256=2f4s2opiHVdrA02ZlBa5pxIHaEjPuG8tdVLsmdII27s,64
|
|
21
|
+
kinto/core/testing.py,sha256=6iKT2q5f1suIavYGdM7lkqbqaIG5h38iM5S6jKU4AQI,5897
|
|
21
22
|
kinto/core/utils.py,sha256=jfP6k63xHNAzbhNkDd9V8YlMzblTyh-ENScETfdW-k0,16772
|
|
22
23
|
kinto/core/cache/__init__.py,sha256=NJT_39WFTuUd-OHuVqgoQTQUYv5zS9PIc3W_Kq9tabc,2726
|
|
23
24
|
kinto/core/cache/memcached.py,sha256=WuYyq6-QykRNryLA_bKdeGAY_Hvwq0c1ejS_8G94cY4,2914
|
|
@@ -77,6 +78,8 @@ kinto/core/views/openapi.py,sha256=YZ7akBoMqmiu0b4AOxlupHU01_kQEHtmzPxJq_iS79Y,9
|
|
|
77
78
|
kinto/core/views/version.py,sha256=-m5G_o0oHTpCgrtfFrHFve6Zqw_gs_szT0Bd8jnNmD4,1419
|
|
78
79
|
kinto/plugins/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
79
80
|
kinto/plugins/flush.py,sha256=HnKJ6-oDo-NMSjdZskqZ09aMnFM-LJHW9vIw4VIyvcI,812
|
|
81
|
+
kinto/plugins/prometheus.py,sha256=ASRMTM_u5oBqHTI4dd5zZUggOCwzcamRAq684BC16TY,4187
|
|
82
|
+
kinto/plugins/statsd.py,sha256=demSOsXdR90T7oDTiYXChlxYICpt4-FVNZHebkH5oo8,1728
|
|
80
83
|
kinto/plugins/accounts/__init__.py,sha256=LUyPfIWIn-HtpmyKO8wAxY9oFsgeYgbMHFmNOcns7rY,4365
|
|
81
84
|
kinto/plugins/accounts/authentication.py,sha256=h7l_KezC8os4sKN1bz3RKEwXwYTYAs21DRzzU9na_U0,3742
|
|
82
85
|
kinto/plugins/accounts/mails.py,sha256=vfb80INjDIJqC1JkPhcwGXlVWjvXhazxA_pnckmOOdo,3481
|
|
@@ -104,13 +107,13 @@ kinto/plugins/admin/build/assets/sql-3IaSLchm.js,sha256=V0dNMfaDvdWL9bgdkSTzMjoG
|
|
|
104
107
|
kinto/plugins/admin/build/assets/ttcn-cfg-9oMIyPXS.js,sha256=8Tr33uTvYVq0BFJHVx62H2FtUMgfuLHjm0GqnqoFjDc,4117
|
|
105
108
|
kinto/plugins/admin/public/help.html,sha256=1hol7z5Sv0Xn3mYyEfPQWFOsrR24htlKhmnGA3rH8fs,958
|
|
106
109
|
kinto/plugins/default_bucket/__init__.py,sha256=Q0frXXJPzFqoPCOjK5OpJZhaz5nCAeR2KwKAP7ltW_U,7287
|
|
107
|
-
kinto/plugins/history/__init__.py,sha256=
|
|
110
|
+
kinto/plugins/history/__init__.py,sha256=s-RMNWaZlmBNd4sNsgJ-hDvMW4pPKUZ-sdZYubb0Kdo,1015
|
|
108
111
|
kinto/plugins/history/listener.py,sha256=Tq5ZHpIOIzQs9yPXA1exEftPoYCuFQJvgxbaIb6XBrM,5033
|
|
109
112
|
kinto/plugins/history/views.py,sha256=NoBP-S7epeH5TLZZbIqfBmwMA2KaWmxP7lqPAS11BTU,2293
|
|
110
113
|
kinto/plugins/openid/__init__.py,sha256=1Iv5SCa6vwEvoJkmGd45-TYm_mxz2okFj6u2VTNuVrk,4863
|
|
111
114
|
kinto/plugins/openid/utils.py,sha256=n3KGS-ogXR2sg6j4QtdPe_DtEsqD7AGVT2K7vhl8yE8,273
|
|
112
115
|
kinto/plugins/openid/views.py,sha256=RVUlS_Ov26NM8SFlrIlpm7ev9OuaMlJTtPA6pUi6ngk,6508
|
|
113
|
-
kinto/plugins/quotas/__init__.py,sha256=
|
|
116
|
+
kinto/plugins/quotas/__init__.py,sha256=itmoP8y4puYYC5CPZAPHhnZ2PXT7PlsAKrNVd0zn4EI,616
|
|
114
117
|
kinto/plugins/quotas/listener.py,sha256=WKKaCNRJw8XoIfnJb3S0ogYhzAo28RcfLE-MzP6Cs2U,8564
|
|
115
118
|
kinto/plugins/quotas/scripts.py,sha256=a3KuzUwtCp_yu78yqr3XdbaHvZyBcUy6o_keNtsuU-M,2941
|
|
116
119
|
kinto/plugins/quotas/utils.py,sha256=BE5bBPrG0BV9Qbcv6-r8WsV7KC6aIJlFeOR4O5tS248,232
|
|
@@ -122,9 +125,9 @@ kinto/views/contribute.py,sha256=NEDr2g1HhVwcMBg0qHEZDmWVJ1V31WsM8cRs0Vm6hfc,118
|
|
|
122
125
|
kinto/views/groups.py,sha256=jOq5fX0-4lwZE8k1q5HME2tU7x9052rtBPF7YqcJ-Qg,3181
|
|
123
126
|
kinto/views/permissions.py,sha256=F0_eKx201WyLonXJ5vLdGKa9RcFKjvAihrEEhU1JuLw,9069
|
|
124
127
|
kinto/views/records.py,sha256=lYfACW2L8qcQoyYBD5IX-fTPjFWmGp7GjHq_U4InlyE,5037
|
|
125
|
-
kinto-
|
|
126
|
-
kinto-
|
|
127
|
-
kinto-
|
|
128
|
-
kinto-
|
|
129
|
-
kinto-
|
|
130
|
-
kinto-
|
|
128
|
+
kinto-19.2.0.dist-info/LICENSE,sha256=oNEIMTuTJzppR5ZEyi86yvvtSagveMYXTYFn56zF0Uk,561
|
|
129
|
+
kinto-19.2.0.dist-info/METADATA,sha256=vRqlFZo0PI2sxAdV3dN9iX7XX52OXJRkbTtYcCZuwAQ,8877
|
|
130
|
+
kinto-19.2.0.dist-info/WHEEL,sha256=OVMc5UfuAQiSplgO0_WdW7vXVGAt9Hdd6qtN4HotdyA,91
|
|
131
|
+
kinto-19.2.0.dist-info/entry_points.txt,sha256=3KlqBWPKY81mrCe_oX0I5s1cRO7Q53nCLbnVr5P9LH4,85
|
|
132
|
+
kinto-19.2.0.dist-info/top_level.txt,sha256=EG_YmbZL6FAug9VwopG7JtF9SvH_r0DEnFp-3twPPys,6
|
|
133
|
+
kinto-19.2.0.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|