kinto 18.1.0__py3-none-any.whl → 20.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of kinto might be problematic. Click here for more details.
- kinto/__init__.py +1 -0
- kinto/__main__.py +1 -19
- kinto/config/kinto.tpl +5 -15
- kinto/contribute.json +27 -0
- kinto/core/__init__.py +21 -8
- kinto/core/cornice/__init__.py +93 -0
- kinto/core/cornice/cors.py +144 -0
- kinto/core/cornice/errors.py +40 -0
- kinto/core/cornice/pyramidhook.py +373 -0
- kinto/core/cornice/renderer.py +89 -0
- kinto/core/cornice/resource.py +205 -0
- kinto/core/cornice/service.py +641 -0
- kinto/core/cornice/util.py +138 -0
- kinto/core/cornice/validators/__init__.py +94 -0
- kinto/core/cornice/validators/_colander.py +142 -0
- kinto/core/cornice/validators/_marshmallow.py +182 -0
- kinto/core/cornice_swagger/__init__.py +92 -0
- kinto/core/cornice_swagger/converters/__init__.py +21 -0
- kinto/core/cornice_swagger/converters/exceptions.py +6 -0
- kinto/core/cornice_swagger/converters/parameters.py +90 -0
- kinto/core/cornice_swagger/converters/schema.py +249 -0
- kinto/core/cornice_swagger/swagger.py +725 -0
- kinto/core/cornice_swagger/templates/index.html +73 -0
- kinto/core/cornice_swagger/templates/index_script_template.html +21 -0
- kinto/core/cornice_swagger/util.py +42 -0
- kinto/core/cornice_swagger/views.py +78 -0
- kinto/core/errors.py +6 -4
- kinto/core/initialization.py +129 -59
- kinto/core/metrics.py +93 -0
- kinto/core/openapi.py +2 -3
- kinto/core/permission/memory.py +3 -2
- kinto/core/permission/postgresql/__init__.py +9 -9
- kinto/core/permission/testing.py +6 -0
- kinto/core/resource/__init__.py +9 -4
- kinto/core/resource/schema.py +1 -2
- kinto/core/resource/viewset.py +1 -1
- kinto/core/statsd.py +1 -63
- kinto/core/storage/__init__.py +15 -0
- kinto/core/storage/memory.py +20 -3
- kinto/core/storage/postgresql/__init__.py +31 -1
- kinto/core/storage/postgresql/client.py +2 -2
- kinto/core/storage/postgresql/migrations/migration_022_023.sql +5 -0
- kinto/core/storage/postgresql/pool.py +1 -1
- kinto/core/storage/postgresql/schema.sql +3 -2
- kinto/core/storage/testing.py +41 -1
- kinto/core/testing.py +6 -2
- kinto/core/utils.py +14 -4
- kinto/core/views/batch.py +1 -1
- kinto/core/views/errors.py +4 -3
- kinto/core/views/openapi.py +1 -1
- kinto/plugins/accounts/__init__.py +3 -21
- kinto/plugins/accounts/authentication.py +8 -54
- kinto/plugins/accounts/utils.py +0 -133
- kinto/plugins/accounts/{views/__init__.py → views.py} +7 -62
- kinto/plugins/admin/VERSION +1 -1
- kinto/plugins/admin/build/VERSION +1 -0
- kinto/plugins/admin/build/assets/asn1-EdZsLKOL.js +1 -0
- kinto/plugins/admin/build/assets/clojure-BMjYHr_A.js +1 -0
- kinto/plugins/admin/build/assets/css-BnMrqG3P.js +1 -0
- kinto/plugins/admin/build/assets/index-Cs7JVwIg.css +6 -0
- kinto/plugins/admin/build/assets/index-CylsivYB.js +165 -0
- kinto/plugins/admin/build/assets/javascript-qCveANmP.js +1 -0
- kinto/plugins/admin/build/assets/logo-VBRiKSPX.png +0 -0
- kinto/plugins/admin/build/assets/mllike-CXdrOF99.js +1 -0
- kinto/plugins/admin/build/assets/python-BuPzkPfP.js +1 -0
- kinto/plugins/admin/build/assets/rpm-CTu-6PCP.js +1 -0
- kinto/plugins/admin/build/assets/sql-D0XecflT.js +1 -0
- kinto/plugins/admin/build/assets/ttcn-cfg-B9xdYoR4.js +1 -0
- kinto/plugins/admin/build/index.html +18 -0
- kinto/plugins/default_bucket/__init__.py +1 -2
- kinto/plugins/flush.py +2 -2
- kinto/plugins/history/__init__.py +15 -6
- kinto/plugins/history/listener.py +68 -5
- kinto/plugins/openid/views.py +1 -1
- kinto/plugins/prometheus.py +203 -0
- kinto/plugins/statsd.py +78 -0
- kinto/views/contribute.py +14 -13
- {kinto-18.1.0.dist-info → kinto-20.4.0.dist-info}/METADATA +31 -32
- kinto-20.4.0.dist-info/RECORD +149 -0
- {kinto-18.1.0.dist-info → kinto-20.4.0.dist-info}/WHEEL +1 -1
- kinto/plugins/accounts/mails.py +0 -96
- kinto/plugins/accounts/views/validation.py +0 -136
- kinto/plugins/quotas/__init__.py +0 -22
- kinto/plugins/quotas/listener.py +0 -226
- kinto/plugins/quotas/scripts.py +0 -80
- kinto/plugins/quotas/utils.py +0 -7
- kinto/scripts.py +0 -41
- kinto-18.1.0.dist-info/RECORD +0 -116
- {kinto-18.1.0.dist-info → kinto-20.4.0.dist-info}/entry_points.txt +0 -0
- {kinto-18.1.0.dist-info → kinto-20.4.0.dist-info/licenses}/LICENSE +0 -0
- {kinto-18.1.0.dist-info → kinto-20.4.0.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
<!-- HTML for static distribution bundle build -->
|
|
2
|
+
<!DOCTYPE html>
|
|
3
|
+
<html lang="en">
|
|
4
|
+
<head>
|
|
5
|
+
<meta charset="UTF-8">
|
|
6
|
+
<title>Swagger UI</title>
|
|
7
|
+
<link href="https://fonts.googleapis.com/css?family=Open+Sans:400,700|Source+Code+Pro:300,600|Titillium+Web:400,600,700" rel="stylesheet">
|
|
8
|
+
<link rel="stylesheet" type="text/css" href="${ui_css_url}" >
|
|
9
|
+
<style>
|
|
10
|
+
html
|
|
11
|
+
{
|
|
12
|
+
box-sizing: border-box;
|
|
13
|
+
overflow: -moz-scrollbars-vertical;
|
|
14
|
+
overflow-y: scroll;
|
|
15
|
+
}
|
|
16
|
+
*,
|
|
17
|
+
*:before,
|
|
18
|
+
*:after
|
|
19
|
+
{
|
|
20
|
+
box-sizing: inherit;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
body {
|
|
24
|
+
margin:0;
|
|
25
|
+
background: #fafafa;
|
|
26
|
+
}
|
|
27
|
+
</style>
|
|
28
|
+
</head>
|
|
29
|
+
|
|
30
|
+
<body>
|
|
31
|
+
|
|
32
|
+
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" style="position:absolute;width:0;height:0">
|
|
33
|
+
<defs>
|
|
34
|
+
<symbol viewBox="0 0 20 20" id="unlocked">
|
|
35
|
+
<path d="M15.8 8H14V5.6C14 2.703 12.665 1 10 1 7.334 1 6 2.703 6 5.6V6h2v-.801C8 3.754 8.797 3 10 3c1.203 0 2 .754 2 2.199V8H4c-.553 0-1 .646-1 1.199V17c0 .549.428 1.139.951 1.307l1.197.387C5.672 18.861 6.55 19 7.1 19h5.8c.549 0 1.428-.139 1.951-.307l1.196-.387c.524-.167.953-.757.953-1.306V9.199C17 8.646 16.352 8 15.8 8z"></path>
|
|
36
|
+
</symbol>
|
|
37
|
+
|
|
38
|
+
<symbol viewBox="0 0 20 20" id="locked">
|
|
39
|
+
<path d="M15.8 8H14V5.6C14 2.703 12.665 1 10 1 7.334 1 6 2.703 6 5.6V8H4c-.553 0-1 .646-1 1.199V17c0 .549.428 1.139.951 1.307l1.197.387C5.672 18.861 6.55 19 7.1 19h5.8c.549 0 1.428-.139 1.951-.307l1.196-.387c.524-.167.953-.757.953-1.306V9.199C17 8.646 16.352 8 15.8 8zM12 8H8V5.199C8 3.754 8.797 3 10 3c1.203 0 2 .754 2 2.199V8z"/>
|
|
40
|
+
</symbol>
|
|
41
|
+
|
|
42
|
+
<symbol viewBox="0 0 20 20" id="close">
|
|
43
|
+
<path d="M14.348 14.849c-.469.469-1.229.469-1.697 0L10 11.819l-2.651 3.029c-.469.469-1.229.469-1.697 0-.469-.469-.469-1.229 0-1.697l2.758-3.15-2.759-3.152c-.469-.469-.469-1.228 0-1.697.469-.469 1.228-.469 1.697 0L10 8.183l2.651-3.031c.469-.469 1.228-.469 1.697 0 .469.469.469 1.229 0 1.697l-2.758 3.152 2.758 3.15c.469.469.469 1.229 0 1.698z"/>
|
|
44
|
+
</symbol>
|
|
45
|
+
|
|
46
|
+
<symbol viewBox="0 0 20 20" id="large-arrow">
|
|
47
|
+
<path d="M13.25 10L6.109 2.58c-.268-.27-.268-.707 0-.979.268-.27.701-.27.969 0l7.83 7.908c.268.271.268.709 0 .979l-7.83 7.908c-.268.271-.701.27-.969 0-.268-.269-.268-.707 0-.979L13.25 10z"/>
|
|
48
|
+
</symbol>
|
|
49
|
+
|
|
50
|
+
<symbol viewBox="0 0 20 20" id="large-arrow-down">
|
|
51
|
+
<path d="M17.418 6.109c.272-.268.709-.268.979 0s.271.701 0 .969l-7.908 7.83c-.27.268-.707.268-.979 0l-7.908-7.83c-.27-.268-.27-.701 0-.969.271-.268.709-.268.979 0L10 13.25l7.418-7.141z"/>
|
|
52
|
+
</symbol>
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
<symbol viewBox="0 0 24 24" id="jump-to">
|
|
56
|
+
<path d="M19 7v4H5.83l3.58-3.59L8 6l-6 6 6 6 1.41-1.41L5.83 13H21V7z"/>
|
|
57
|
+
</symbol>
|
|
58
|
+
|
|
59
|
+
<symbol viewBox="0 0 24 24" id="expand">
|
|
60
|
+
<path d="M10 18h4v-2h-4v2zM3 6v2h18V6H3zm3 7h12v-2H6v2z"/>
|
|
61
|
+
</symbol>
|
|
62
|
+
|
|
63
|
+
</defs>
|
|
64
|
+
</svg>
|
|
65
|
+
|
|
66
|
+
<div id="swagger-ui"></div>
|
|
67
|
+
|
|
68
|
+
<script src="${ui_js_bundle_url}"> </script>
|
|
69
|
+
<script src="${ui_js_standalone_url}"> </script>
|
|
70
|
+
${swagger_ui_script}
|
|
71
|
+
</body>
|
|
72
|
+
|
|
73
|
+
</html>
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
<script>
|
|
2
|
+
window.onload = function() {
|
|
3
|
+
|
|
4
|
+
// Build a system
|
|
5
|
+
const ui = SwaggerUIBundle({
|
|
6
|
+
url: "${swagger_spec_url}",
|
|
7
|
+
dom_id: '#swagger-ui',
|
|
8
|
+
deepLinking: true,
|
|
9
|
+
presets: [
|
|
10
|
+
SwaggerUIBundle.presets.apis,
|
|
11
|
+
SwaggerUIStandalonePreset
|
|
12
|
+
],
|
|
13
|
+
plugins: [
|
|
14
|
+
SwaggerUIBundle.plugins.DownloadUrl
|
|
15
|
+
],
|
|
16
|
+
layout: "StandaloneLayout"
|
|
17
|
+
})
|
|
18
|
+
|
|
19
|
+
window.ui = ui
|
|
20
|
+
}
|
|
21
|
+
</script>
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import colander
|
|
2
|
+
|
|
3
|
+
from kinto.core.cornice.validators import colander_body_validator
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def trim(docstring):
|
|
7
|
+
"""
|
|
8
|
+
Remove the tabs to spaces, and remove the extra spaces / tabs that are in
|
|
9
|
+
front of the text in docstrings.
|
|
10
|
+
|
|
11
|
+
Implementation taken from http://www.python.org/dev/peps/pep-0257/
|
|
12
|
+
"""
|
|
13
|
+
if not docstring:
|
|
14
|
+
return ""
|
|
15
|
+
# Convert tabs to spaces (following the normal Python rules)
|
|
16
|
+
# and split into a list of lines:
|
|
17
|
+
lines = docstring.expandtabs().splitlines()
|
|
18
|
+
lines = [line.strip() for line in lines]
|
|
19
|
+
res = "\n".join(lines)
|
|
20
|
+
return res
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def body_schema_transformer(schema, args):
|
|
24
|
+
validators = args.get("validators", [])
|
|
25
|
+
if colander_body_validator in validators:
|
|
26
|
+
body_schema = schema
|
|
27
|
+
schema = colander.MappingSchema()
|
|
28
|
+
schema["body"] = body_schema
|
|
29
|
+
return schema
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def merge_dicts(base, changes):
|
|
33
|
+
"""Merge b into a recursively, without overwriting values.
|
|
34
|
+
|
|
35
|
+
:param base: the dict that will be altered.
|
|
36
|
+
:param changes: changes to update base.
|
|
37
|
+
"""
|
|
38
|
+
for k, v in changes.items():
|
|
39
|
+
if isinstance(v, dict):
|
|
40
|
+
merge_dicts(base.setdefault(k, {}), v)
|
|
41
|
+
else:
|
|
42
|
+
base.setdefault(k, v)
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import importlib
|
|
2
|
+
from string import Template
|
|
3
|
+
|
|
4
|
+
import cornice
|
|
5
|
+
import cornice_swagger
|
|
6
|
+
import pkg_resources
|
|
7
|
+
from pyramid.response import Response
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
# hardcode for now since that will work for vast majority of users
|
|
11
|
+
# maybe later add minified resources for behind firewall support?
|
|
12
|
+
ui_css_url = "https://cdnjs.cloudflare.com/ajax/libs/swagger-ui/3.23.11/swagger-ui.css"
|
|
13
|
+
ui_js_bundle_url = "https://cdnjs.cloudflare.com/ajax/libs/swagger-ui/3.23.11/swagger-ui-bundle.js"
|
|
14
|
+
ui_js_standalone_url = (
|
|
15
|
+
"https://cdnjs.cloudflare.com/ajax/libs/swagger-ui/3.23.11/swagger-ui-standalone-preset.js"
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def swagger_ui_template_view(request):
|
|
20
|
+
"""
|
|
21
|
+
Serves Swagger UI page, default Swagger UI config is used but you can
|
|
22
|
+
override the callable that generates the `<script>` tag by setting
|
|
23
|
+
`cornice_swagger.swagger_ui_script_generator` in pyramid config, it defaults
|
|
24
|
+
to 'cornice_swagger.views:swagger_ui_script_template'
|
|
25
|
+
|
|
26
|
+
:param request:
|
|
27
|
+
:return:
|
|
28
|
+
"""
|
|
29
|
+
script_generator = request.registry.settings.get(
|
|
30
|
+
"cornice_swagger.swagger_ui_script_generator",
|
|
31
|
+
"cornice_swagger.views:swagger_ui_script_template",
|
|
32
|
+
)
|
|
33
|
+
package, callable = script_generator.split(":")
|
|
34
|
+
imported_package = importlib.import_module(package)
|
|
35
|
+
script_callable = getattr(imported_package, callable)
|
|
36
|
+
template = pkg_resources.resource_string("cornice_swagger", "templates/index.html").decode(
|
|
37
|
+
"utf8"
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
html = Template(template).safe_substitute(
|
|
41
|
+
ui_css_url=ui_css_url,
|
|
42
|
+
ui_js_bundle_url=ui_js_bundle_url,
|
|
43
|
+
ui_js_standalone_url=ui_js_standalone_url,
|
|
44
|
+
swagger_ui_script=script_callable(request),
|
|
45
|
+
)
|
|
46
|
+
return Response(html)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def open_api_json_view(request):
|
|
50
|
+
"""
|
|
51
|
+
:param request:
|
|
52
|
+
:return:
|
|
53
|
+
|
|
54
|
+
Generates JSON representation of Swagger spec
|
|
55
|
+
"""
|
|
56
|
+
doc = cornice_swagger.CorniceSwagger(
|
|
57
|
+
cornice.service.get_services(), pyramid_registry=request.registry
|
|
58
|
+
)
|
|
59
|
+
kwargs = request.registry.settings["cornice_swagger.spec_kwargs"]
|
|
60
|
+
my_spec = doc.generate(**kwargs)
|
|
61
|
+
return my_spec
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def swagger_ui_script_template(request, **kwargs):
|
|
65
|
+
"""
|
|
66
|
+
:param request:
|
|
67
|
+
:return:
|
|
68
|
+
|
|
69
|
+
Generates the <script> code that bootstraps Swagger UI, it will be injected
|
|
70
|
+
into index template
|
|
71
|
+
"""
|
|
72
|
+
swagger_spec_url = request.route_url("cornice_swagger.open_api_path")
|
|
73
|
+
template = pkg_resources.resource_string(
|
|
74
|
+
"cornice_swagger", "templates/index_script_template.html"
|
|
75
|
+
).decode("utf8")
|
|
76
|
+
return Template(template).safe_substitute(
|
|
77
|
+
swagger_spec_url=swagger_spec_url,
|
|
78
|
+
)
|
kinto/core/errors.py
CHANGED
|
@@ -140,14 +140,16 @@ def json_error_handler(request):
|
|
|
140
140
|
"""
|
|
141
141
|
errors = request.errors
|
|
142
142
|
sorted_errors = sorted(errors, key=lambda x: str(x["name"]))
|
|
143
|
+
for error in sorted_errors:
|
|
144
|
+
# Decode in place.
|
|
145
|
+
if isinstance(error["description"], bytes):
|
|
146
|
+
error["description"] = error["description"].decode("utf-8")
|
|
147
|
+
|
|
143
148
|
# In Cornice, we call error handler if at least one error was set.
|
|
144
149
|
error = sorted_errors[0]
|
|
145
150
|
name = error["name"]
|
|
146
151
|
description = error["description"]
|
|
147
152
|
|
|
148
|
-
if isinstance(description, bytes):
|
|
149
|
-
description = error["description"].decode("utf-8")
|
|
150
|
-
|
|
151
153
|
if name is not None:
|
|
152
154
|
if str(name) in description:
|
|
153
155
|
message = description
|
|
@@ -162,7 +164,7 @@ def json_error_handler(request):
|
|
|
162
164
|
errno=ERRORS.INVALID_PARAMETERS.value,
|
|
163
165
|
error="Invalid parameters",
|
|
164
166
|
message=message,
|
|
165
|
-
details=
|
|
167
|
+
details=sorted_errors,
|
|
166
168
|
)
|
|
167
169
|
response.status = errors.status
|
|
168
170
|
response = reapply_cors(request, response)
|
kinto/core/initialization.py
CHANGED
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
import logging
|
|
2
2
|
import random
|
|
3
3
|
import re
|
|
4
|
+
import urllib.parse
|
|
4
5
|
import warnings
|
|
5
6
|
from datetime import datetime
|
|
6
|
-
from secrets import token_hex
|
|
7
7
|
|
|
8
8
|
from dateutil import parser as dateparser
|
|
9
|
+
from dockerflow.logging import get_or_generate_request_id, request_id_context
|
|
9
10
|
from pyramid.events import ApplicationCreated, NewRequest, NewResponse
|
|
10
11
|
from pyramid.exceptions import ConfigurationError
|
|
11
12
|
from pyramid.httpexceptions import (
|
|
@@ -21,7 +22,7 @@ from pyramid.security import NO_PERMISSION_REQUIRED
|
|
|
21
22
|
from pyramid.settings import asbool, aslist
|
|
22
23
|
from pyramid_multiauth import MultiAuthenticationPolicy, MultiAuthPolicySelected
|
|
23
24
|
|
|
24
|
-
from kinto.core import cache, errors, permission, storage, utils
|
|
25
|
+
from kinto.core import cache, errors, metrics, permission, storage, utils
|
|
25
26
|
from kinto.core.events import ACTIONS, ResourceChanged, ResourceRead
|
|
26
27
|
|
|
27
28
|
|
|
@@ -212,7 +213,7 @@ def setup_deprecation(config):
|
|
|
212
213
|
|
|
213
214
|
def _end_of_life_tween_factory(handler, registry):
|
|
214
215
|
"""Pyramid tween to handle service end of life."""
|
|
215
|
-
deprecation_msg = "The service you are trying to connect no longer exists
|
|
216
|
+
deprecation_msg = "The service you are trying to connect no longer exists at this location."
|
|
216
217
|
|
|
217
218
|
def eos_tween(request):
|
|
218
219
|
eos_date = registry.settings["eos"]
|
|
@@ -333,54 +334,6 @@ def setup_sentry(config):
|
|
|
333
334
|
config.add_subscriber(on_app_created, ApplicationCreated)
|
|
334
335
|
|
|
335
336
|
|
|
336
|
-
def setup_statsd(config):
|
|
337
|
-
settings = config.get_settings()
|
|
338
|
-
config.registry.statsd = None
|
|
339
|
-
|
|
340
|
-
if settings["statsd_url"]:
|
|
341
|
-
statsd_mod = settings["statsd_backend"]
|
|
342
|
-
statsd_mod = config.maybe_dotted(statsd_mod)
|
|
343
|
-
client = statsd_mod.load_from_config(config)
|
|
344
|
-
|
|
345
|
-
config.registry.statsd = client
|
|
346
|
-
|
|
347
|
-
client.watch_execution_time(config.registry.cache, prefix="backend")
|
|
348
|
-
client.watch_execution_time(config.registry.storage, prefix="backend")
|
|
349
|
-
client.watch_execution_time(config.registry.permission, prefix="backend")
|
|
350
|
-
|
|
351
|
-
# Commit so that configured policy can be queried.
|
|
352
|
-
config.commit()
|
|
353
|
-
policy = config.registry.queryUtility(IAuthenticationPolicy)
|
|
354
|
-
if isinstance(policy, MultiAuthenticationPolicy):
|
|
355
|
-
for name, subpolicy in policy.get_policies():
|
|
356
|
-
client.watch_execution_time(subpolicy, prefix="authentication", classname=name)
|
|
357
|
-
else:
|
|
358
|
-
client.watch_execution_time(policy, prefix="authentication")
|
|
359
|
-
|
|
360
|
-
def on_new_response(event):
|
|
361
|
-
request = event.request
|
|
362
|
-
|
|
363
|
-
# Count unique users.
|
|
364
|
-
user_id = request.prefixed_userid
|
|
365
|
-
if user_id:
|
|
366
|
-
# Get rid of colons in metric packet (see #1282).
|
|
367
|
-
user_id = user_id.replace(":", ".")
|
|
368
|
-
client.count("users", unique=user_id)
|
|
369
|
-
|
|
370
|
-
# Count authentication verifications.
|
|
371
|
-
if hasattr(request, "authn_type"):
|
|
372
|
-
client.count(f"authn_type.{request.authn_type}")
|
|
373
|
-
|
|
374
|
-
# Count view calls.
|
|
375
|
-
service = request.current_service
|
|
376
|
-
if service:
|
|
377
|
-
client.count(f"view.{service.name}.{request.method}")
|
|
378
|
-
|
|
379
|
-
config.add_subscriber(on_new_response, NewResponse)
|
|
380
|
-
|
|
381
|
-
return client
|
|
382
|
-
|
|
383
|
-
|
|
384
337
|
def install_middlewares(app, settings):
|
|
385
338
|
"Install a set of middlewares defined in the ini file on the given app."
|
|
386
339
|
# Setup new-relic.
|
|
@@ -421,12 +374,15 @@ def setup_logging(config):
|
|
|
421
374
|
message="Invalid URL path.",
|
|
422
375
|
)
|
|
423
376
|
|
|
377
|
+
rid = get_or_generate_request_id(headers=request.headers)
|
|
378
|
+
request_id_context.set(rid)
|
|
379
|
+
|
|
424
380
|
request.log_context(
|
|
425
381
|
agent=request.headers.get("User-Agent"),
|
|
426
382
|
path=request_path,
|
|
427
383
|
method=request.method,
|
|
428
384
|
lang=request.headers.get("Accept-Language"),
|
|
429
|
-
rid=
|
|
385
|
+
rid=rid,
|
|
430
386
|
errno=0,
|
|
431
387
|
)
|
|
432
388
|
qs = dict(errors.request_GET(request))
|
|
@@ -466,6 +422,122 @@ def setup_logging(config):
|
|
|
466
422
|
config.add_subscriber(on_new_response, NewResponse)
|
|
467
423
|
|
|
468
424
|
|
|
425
|
+
def setup_metrics(config):
|
|
426
|
+
settings = config.get_settings()
|
|
427
|
+
|
|
428
|
+
# Register a no-op metrics service by default.
|
|
429
|
+
config.registry.registerUtility(metrics.NoOpMetricsService(), metrics.IMetricsService)
|
|
430
|
+
|
|
431
|
+
# This does not fully respect the Pyramid/ZCA patterns, but the rest of Kinto uses
|
|
432
|
+
# `registry.storage`, `registry.cache`, etc. Consistency seems more important.
|
|
433
|
+
config.registry.__class__.metrics = property(
|
|
434
|
+
lambda reg: reg.queryUtility(metrics.IMetricsService)
|
|
435
|
+
)
|
|
436
|
+
|
|
437
|
+
def deprecated_registry(self):
|
|
438
|
+
warnings.warn(
|
|
439
|
+
"``config.registry.statsd`` is now deprecated. Use ``config.registry.metrics`` instead.",
|
|
440
|
+
DeprecationWarning,
|
|
441
|
+
)
|
|
442
|
+
return self.metrics
|
|
443
|
+
|
|
444
|
+
config.registry.__class__.statsd = property(deprecated_registry)
|
|
445
|
+
|
|
446
|
+
def on_app_created(event):
|
|
447
|
+
config = event.app
|
|
448
|
+
metrics_service = config.registry.metrics
|
|
449
|
+
|
|
450
|
+
metrics.watch_execution_time(metrics_service, config.registry.cache, prefix="backend")
|
|
451
|
+
metrics.watch_execution_time(metrics_service, config.registry.storage, prefix="backend")
|
|
452
|
+
metrics.watch_execution_time(metrics_service, config.registry.permission, prefix="backend")
|
|
453
|
+
|
|
454
|
+
policy = config.registry.queryUtility(IAuthenticationPolicy)
|
|
455
|
+
if isinstance(policy, MultiAuthenticationPolicy):
|
|
456
|
+
for name, subpolicy in policy.get_policies():
|
|
457
|
+
metrics.watch_execution_time(
|
|
458
|
+
metrics_service, subpolicy, prefix="authentication", classname=name
|
|
459
|
+
)
|
|
460
|
+
else:
|
|
461
|
+
metrics.watch_execution_time(metrics_service, policy, prefix="authentication")
|
|
462
|
+
|
|
463
|
+
config.add_subscriber(on_app_created, ApplicationCreated)
|
|
464
|
+
|
|
465
|
+
def on_new_response(event):
|
|
466
|
+
request = event.request
|
|
467
|
+
metrics_service = config.registry.metrics
|
|
468
|
+
|
|
469
|
+
try:
|
|
470
|
+
endpoint = utils.strip_uri_prefix(request.path)
|
|
471
|
+
endpoint = urllib.parse.quote_plus(endpoint, safe="/?&=-_")
|
|
472
|
+
except UnicodeDecodeError as e:
|
|
473
|
+
# This `on_new_response` callback is also called when a HTTP 400
|
|
474
|
+
# is returned because of an invalid UTF-8 path. We still want metrics.
|
|
475
|
+
endpoint = str(e)
|
|
476
|
+
|
|
477
|
+
# Count unique users.
|
|
478
|
+
user_id = request.prefixed_userid
|
|
479
|
+
if user_id:
|
|
480
|
+
# Get rid of colons in metric packet (see #1282).
|
|
481
|
+
auth, user_id = user_id.split(":")
|
|
482
|
+
metrics_service.count("users", unique=[("auth", auth), ("userid", user_id)])
|
|
483
|
+
|
|
484
|
+
# Add extra labels to metrics, based on fields extracted from the request matchdict.
|
|
485
|
+
metrics_matchdict_fields = aslist(settings["metrics_matchdict_fields"])
|
|
486
|
+
# Turn the `id` field of object endpoints into `{resource}_id` (eg. `mushroom_id`, `bucket_id`)
|
|
487
|
+
enhanced_matchdict = dict(request.matchdict or {})
|
|
488
|
+
try:
|
|
489
|
+
enhanced_matchdict[request.current_resource_name + "_id"] = enhanced_matchdict.get(
|
|
490
|
+
"id", ""
|
|
491
|
+
)
|
|
492
|
+
except AttributeError:
|
|
493
|
+
# Not on a resource.
|
|
494
|
+
pass
|
|
495
|
+
metrics_matchdict_labels = [
|
|
496
|
+
(field, enhanced_matchdict.get(field, "")) for field in metrics_matchdict_fields
|
|
497
|
+
]
|
|
498
|
+
|
|
499
|
+
# Count served requests.
|
|
500
|
+
metrics_service.count(
|
|
501
|
+
"request_summary",
|
|
502
|
+
unique=[
|
|
503
|
+
("method", request.method.lower()),
|
|
504
|
+
("endpoint", endpoint),
|
|
505
|
+
("status", str(event.response.status_code)),
|
|
506
|
+
]
|
|
507
|
+
+ metrics_matchdict_labels,
|
|
508
|
+
)
|
|
509
|
+
|
|
510
|
+
try:
|
|
511
|
+
current = utils.msec_time()
|
|
512
|
+
duration = current - request._received_at
|
|
513
|
+
metrics_service.observe(
|
|
514
|
+
"request_duration",
|
|
515
|
+
duration,
|
|
516
|
+
labels=[("endpoint", endpoint)] + metrics_matchdict_labels,
|
|
517
|
+
)
|
|
518
|
+
except AttributeError: # pragma: no cover
|
|
519
|
+
# Logging was not setup in this Kinto app (unlikely but possible)
|
|
520
|
+
pass
|
|
521
|
+
|
|
522
|
+
# Observe response size.
|
|
523
|
+
metrics_service.observe(
|
|
524
|
+
"request_size",
|
|
525
|
+
len(event.response.body or b""),
|
|
526
|
+
labels=[("endpoint", endpoint)] + metrics_matchdict_labels,
|
|
527
|
+
)
|
|
528
|
+
|
|
529
|
+
# Count authentication verifications.
|
|
530
|
+
if hasattr(request, "authn_type"):
|
|
531
|
+
metrics_service.count(f"authn_type.{request.authn_type}")
|
|
532
|
+
|
|
533
|
+
# Count view calls.
|
|
534
|
+
service = request.current_service
|
|
535
|
+
if service:
|
|
536
|
+
metrics_service.count(f"view.{service.name}.{request.method}")
|
|
537
|
+
|
|
538
|
+
config.add_subscriber(on_new_response, NewResponse)
|
|
539
|
+
|
|
540
|
+
|
|
469
541
|
class EventActionFilter:
|
|
470
542
|
def __init__(self, actions, config):
|
|
471
543
|
actions = ACTIONS.from_string_list(actions)
|
|
@@ -518,11 +590,9 @@ def setup_listeners(config):
|
|
|
518
590
|
listener_mod = config.maybe_dotted(module_value)
|
|
519
591
|
listener = listener_mod.load_from_config(config, prefix)
|
|
520
592
|
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
key = f"listeners.{name}"
|
|
525
|
-
listener = statsd_client.timer(key)(listener.__call__)
|
|
593
|
+
wrapped_listener = metrics.listener_with_timer(
|
|
594
|
+
config, f"listeners.{name}", listener.__call__
|
|
595
|
+
)
|
|
526
596
|
|
|
527
597
|
# Optional filter by event action.
|
|
528
598
|
actions_setting = prefix + "actions"
|
|
@@ -548,11 +618,11 @@ def setup_listeners(config):
|
|
|
548
618
|
options = dict(for_actions=actions, for_resources=resource_names)
|
|
549
619
|
|
|
550
620
|
if ACTIONS.READ in actions:
|
|
551
|
-
config.add_subscriber(
|
|
621
|
+
config.add_subscriber(wrapped_listener, ResourceRead, **options)
|
|
552
622
|
actions = [a for a in actions if a != ACTIONS.READ]
|
|
553
623
|
|
|
554
624
|
if len(actions) > 0:
|
|
555
|
-
config.add_subscriber(
|
|
625
|
+
config.add_subscriber(wrapped_listener, ResourceChanged, **options)
|
|
556
626
|
|
|
557
627
|
|
|
558
628
|
def load_default_settings(config, default_settings):
|
kinto/core/metrics.py
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import types
|
|
2
|
+
|
|
3
|
+
from zope.interface import Interface, implementer
|
|
4
|
+
|
|
5
|
+
from kinto.core import utils
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class IMetricsService(Interface):
|
|
9
|
+
"""
|
|
10
|
+
An interface that defines the metrics service contract.
|
|
11
|
+
Any class implementing this must provide all its methods.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
def timer(key):
|
|
15
|
+
"""
|
|
16
|
+
Watch execution time.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
def observe(self, key, value, labels=[]):
|
|
20
|
+
"""
|
|
21
|
+
Observe a give `value` for the specified `key`.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
def count(key, count=1, unique=None):
|
|
25
|
+
"""
|
|
26
|
+
Count occurrences. If `unique` is set, overwrites the counter value
|
|
27
|
+
on each call.
|
|
28
|
+
|
|
29
|
+
`unique` should be of type ``list[tuple[str,str]]``.
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class NoOpTimer:
|
|
34
|
+
def __call__(self, f):
|
|
35
|
+
@utils.safe_wraps(f)
|
|
36
|
+
def _wrapped(*args, **kwargs):
|
|
37
|
+
return f(*args, **kwargs)
|
|
38
|
+
|
|
39
|
+
return _wrapped
|
|
40
|
+
|
|
41
|
+
def __enter__(self):
|
|
42
|
+
pass
|
|
43
|
+
|
|
44
|
+
def __exit__(self, *args, **kwargs):
|
|
45
|
+
pass
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@implementer(IMetricsService)
|
|
49
|
+
class NoOpMetricsService:
|
|
50
|
+
def timer(self, key):
|
|
51
|
+
return NoOpTimer()
|
|
52
|
+
|
|
53
|
+
def observe(self, key, value, labels=[]):
|
|
54
|
+
pass
|
|
55
|
+
|
|
56
|
+
def count(self, key, count=1, unique=None):
|
|
57
|
+
pass
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def watch_execution_time(metrics_service, obj, prefix="", classname=None):
|
|
61
|
+
"""
|
|
62
|
+
Decorate all methods of an object in order to watch their execution time.
|
|
63
|
+
Metrics will be named `{prefix}.{classname}.{method}`.
|
|
64
|
+
"""
|
|
65
|
+
classname = classname or utils.classname(obj)
|
|
66
|
+
members = dir(obj)
|
|
67
|
+
for name in members:
|
|
68
|
+
value = getattr(obj, name)
|
|
69
|
+
is_method = isinstance(value, types.MethodType)
|
|
70
|
+
if not name.startswith("_") and is_method:
|
|
71
|
+
statsd_key = f"{prefix}.{classname}.{name}"
|
|
72
|
+
decorated_method = metrics_service.timer(statsd_key)(value)
|
|
73
|
+
setattr(obj, name, decorated_method)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def listener_with_timer(config, key, func):
|
|
77
|
+
"""
|
|
78
|
+
Add a timer with the specified `key` on the specified `func`.
|
|
79
|
+
This is used to avoid evaluating `config.registry.metrics` during setup time
|
|
80
|
+
to avoid having to deal with initialization order and configuration committing.
|
|
81
|
+
"""
|
|
82
|
+
|
|
83
|
+
def wrapped(*args, **kwargs):
|
|
84
|
+
metrics_service = config.registry.metrics
|
|
85
|
+
if not metrics_service:
|
|
86
|
+
# This only happens if `kinto.core.initialization.setup_metrics` is
|
|
87
|
+
# not listed in the `initialization_sequence` setting.
|
|
88
|
+
return func(*args, **kwargs)
|
|
89
|
+
# If metrics are enabled, monitor execution time of listeners.
|
|
90
|
+
with metrics_service.timer(key):
|
|
91
|
+
return func(*args, **kwargs)
|
|
92
|
+
|
|
93
|
+
return wrapped
|
kinto/core/openapi.py
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
|
-
from cornice_swagger import CorniceSwagger
|
|
2
|
-
from cornice_swagger.converters.schema import TypeConverter
|
|
3
|
-
|
|
1
|
+
from kinto.core.cornice_swagger import CorniceSwagger
|
|
2
|
+
from kinto.core.cornice_swagger.converters.schema import TypeConverter
|
|
4
3
|
from kinto.core.schema import Any
|
|
5
4
|
|
|
6
5
|
|
kinto/core/permission/memory.py
CHANGED
|
@@ -98,8 +98,9 @@ class Permission(PermissionBase):
|
|
|
98
98
|
candidates = []
|
|
99
99
|
if bound_permissions is None:
|
|
100
100
|
for key, value in self._store.items():
|
|
101
|
-
|
|
102
|
-
|
|
101
|
+
if key.startswith("permission:"):
|
|
102
|
+
_, object_id, permission = key.split(":", 2)
|
|
103
|
+
candidates.append((object_id, permission, value))
|
|
103
104
|
else:
|
|
104
105
|
for pattern, perm in bound_permissions:
|
|
105
106
|
id_match = ".*" if with_children else "[^/]+"
|
|
@@ -246,7 +246,7 @@ class Permission(PermissionBase, MigratorMixin):
|
|
|
246
246
|
|
|
247
247
|
query = f"""
|
|
248
248
|
WITH required_perms AS (
|
|
249
|
-
VALUES {
|
|
249
|
+
VALUES {",".join(perm_values)}
|
|
250
250
|
)
|
|
251
251
|
SELECT principal
|
|
252
252
|
FROM required_perms JOIN access_control_entries
|
|
@@ -292,14 +292,14 @@ class Permission(PermissionBase, MigratorMixin):
|
|
|
292
292
|
object_id_condition = "object_id LIKE pattern"
|
|
293
293
|
else:
|
|
294
294
|
object_id_condition = (
|
|
295
|
-
"object_id LIKE pattern
|
|
295
|
+
"object_id LIKE pattern AND object_id NOT LIKE pattern || '/%'"
|
|
296
296
|
)
|
|
297
297
|
query = f"""
|
|
298
298
|
WITH required_perms AS (
|
|
299
|
-
VALUES {
|
|
299
|
+
VALUES {",".join(perm_values)}
|
|
300
300
|
),
|
|
301
301
|
user_principals AS (
|
|
302
|
-
VALUES {
|
|
302
|
+
VALUES {",".join(principals_values)}
|
|
303
303
|
),
|
|
304
304
|
potential_objects AS (
|
|
305
305
|
SELECT object_id, permission, required_perms.column1 AS pattern
|
|
@@ -341,7 +341,7 @@ class Permission(PermissionBase, MigratorMixin):
|
|
|
341
341
|
|
|
342
342
|
query = f"""
|
|
343
343
|
WITH required_perms AS (
|
|
344
|
-
VALUES {
|
|
344
|
+
VALUES {",".join(perms_values)}
|
|
345
345
|
),
|
|
346
346
|
allowed_principals AS (
|
|
347
347
|
SELECT principal
|
|
@@ -349,7 +349,7 @@ class Permission(PermissionBase, MigratorMixin):
|
|
|
349
349
|
ON (object_id = column1 AND permission = column2)
|
|
350
350
|
),
|
|
351
351
|
required_principals AS (
|
|
352
|
-
VALUES {
|
|
352
|
+
VALUES {",".join(principals_values)}
|
|
353
353
|
)
|
|
354
354
|
SELECT COUNT(*) AS matched
|
|
355
355
|
FROM required_principals JOIN allowed_principals
|
|
@@ -412,7 +412,7 @@ class Permission(PermissionBase, MigratorMixin):
|
|
|
412
412
|
if not new_aces:
|
|
413
413
|
query = f"""
|
|
414
414
|
WITH specified_perms AS (
|
|
415
|
-
VALUES {
|
|
415
|
+
VALUES {",".join(specified_perms)}
|
|
416
416
|
)
|
|
417
417
|
DELETE FROM access_control_entries
|
|
418
418
|
USING specified_perms
|
|
@@ -422,7 +422,7 @@ class Permission(PermissionBase, MigratorMixin):
|
|
|
422
422
|
else:
|
|
423
423
|
query = f"""
|
|
424
424
|
WITH specified_perms AS (
|
|
425
|
-
VALUES {
|
|
425
|
+
VALUES {",".join(specified_perms)}
|
|
426
426
|
),
|
|
427
427
|
delete_specified AS (
|
|
428
428
|
DELETE FROM access_control_entries
|
|
@@ -435,7 +435,7 @@ class Permission(PermissionBase, MigratorMixin):
|
|
|
435
435
|
UNION SELECT :object_id
|
|
436
436
|
),
|
|
437
437
|
new_aces AS (
|
|
438
|
-
VALUES {
|
|
438
|
+
VALUES {",".join(new_aces)}
|
|
439
439
|
)
|
|
440
440
|
INSERT INTO access_control_entries(object_id, permission, principal)
|
|
441
441
|
SELECT DISTINCT d.object_id, n.column1, n.column2
|