kinto 23.2.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kinto/__init__.py +92 -0
- kinto/__main__.py +249 -0
- kinto/authorization.py +134 -0
- kinto/config/__init__.py +94 -0
- kinto/config/kinto.tpl +270 -0
- kinto/contribute.json +27 -0
- kinto/core/__init__.py +246 -0
- kinto/core/authentication.py +48 -0
- kinto/core/authorization.py +311 -0
- kinto/core/cache/__init__.py +131 -0
- kinto/core/cache/memcached.py +112 -0
- kinto/core/cache/memory.py +104 -0
- kinto/core/cache/postgresql/__init__.py +178 -0
- kinto/core/cache/postgresql/schema.sql +23 -0
- kinto/core/cache/testing.py +208 -0
- kinto/core/cornice/__init__.py +93 -0
- kinto/core/cornice/cors.py +144 -0
- kinto/core/cornice/errors.py +40 -0
- kinto/core/cornice/pyramidhook.py +373 -0
- kinto/core/cornice/renderer.py +89 -0
- kinto/core/cornice/resource.py +205 -0
- kinto/core/cornice/service.py +641 -0
- kinto/core/cornice/util.py +138 -0
- kinto/core/cornice/validators/__init__.py +94 -0
- kinto/core/cornice/validators/_colander.py +142 -0
- kinto/core/cornice/validators/_marshmallow.py +182 -0
- kinto/core/cornice_swagger/__init__.py +92 -0
- kinto/core/cornice_swagger/converters/__init__.py +21 -0
- kinto/core/cornice_swagger/converters/exceptions.py +6 -0
- kinto/core/cornice_swagger/converters/parameters.py +90 -0
- kinto/core/cornice_swagger/converters/schema.py +249 -0
- kinto/core/cornice_swagger/swagger.py +725 -0
- kinto/core/cornice_swagger/templates/index.html +73 -0
- kinto/core/cornice_swagger/templates/index_script_template.html +21 -0
- kinto/core/cornice_swagger/util.py +42 -0
- kinto/core/cornice_swagger/views.py +78 -0
- kinto/core/decorators.py +74 -0
- kinto/core/errors.py +216 -0
- kinto/core/events.py +301 -0
- kinto/core/initialization.py +738 -0
- kinto/core/listeners/__init__.py +9 -0
- kinto/core/metrics.py +94 -0
- kinto/core/openapi.py +115 -0
- kinto/core/permission/__init__.py +202 -0
- kinto/core/permission/memory.py +167 -0
- kinto/core/permission/postgresql/__init__.py +489 -0
- kinto/core/permission/postgresql/migrations/migration_001_002.sql +18 -0
- kinto/core/permission/postgresql/schema.sql +41 -0
- kinto/core/permission/testing.py +487 -0
- kinto/core/resource/__init__.py +1311 -0
- kinto/core/resource/model.py +412 -0
- kinto/core/resource/schema.py +502 -0
- kinto/core/resource/viewset.py +230 -0
- kinto/core/schema.py +119 -0
- kinto/core/scripts.py +50 -0
- kinto/core/statsd.py +1 -0
- kinto/core/storage/__init__.py +436 -0
- kinto/core/storage/exceptions.py +53 -0
- kinto/core/storage/generators.py +58 -0
- kinto/core/storage/memory.py +651 -0
- kinto/core/storage/postgresql/__init__.py +1131 -0
- kinto/core/storage/postgresql/client.py +120 -0
- kinto/core/storage/postgresql/migrations/migration_001_002.sql +10 -0
- kinto/core/storage/postgresql/migrations/migration_002_003.sql +33 -0
- kinto/core/storage/postgresql/migrations/migration_003_004.sql +18 -0
- kinto/core/storage/postgresql/migrations/migration_004_005.sql +20 -0
- kinto/core/storage/postgresql/migrations/migration_005_006.sql +11 -0
- kinto/core/storage/postgresql/migrations/migration_006_007.sql +74 -0
- kinto/core/storage/postgresql/migrations/migration_007_008.sql +66 -0
- kinto/core/storage/postgresql/migrations/migration_008_009.sql +41 -0
- kinto/core/storage/postgresql/migrations/migration_009_010.sql +98 -0
- kinto/core/storage/postgresql/migrations/migration_010_011.sql +14 -0
- kinto/core/storage/postgresql/migrations/migration_011_012.sql +9 -0
- kinto/core/storage/postgresql/migrations/migration_012_013.sql +71 -0
- kinto/core/storage/postgresql/migrations/migration_013_014.sql +14 -0
- kinto/core/storage/postgresql/migrations/migration_014_015.sql +95 -0
- kinto/core/storage/postgresql/migrations/migration_015_016.sql +4 -0
- kinto/core/storage/postgresql/migrations/migration_016_017.sql +81 -0
- kinto/core/storage/postgresql/migrations/migration_017_018.sql +25 -0
- kinto/core/storage/postgresql/migrations/migration_018_019.sql +8 -0
- kinto/core/storage/postgresql/migrations/migration_019_020.sql +7 -0
- kinto/core/storage/postgresql/migrations/migration_020_021.sql +68 -0
- kinto/core/storage/postgresql/migrations/migration_021_022.sql +62 -0
- kinto/core/storage/postgresql/migrations/migration_022_023.sql +5 -0
- kinto/core/storage/postgresql/migrations/migration_023_024.sql +6 -0
- kinto/core/storage/postgresql/migrations/migration_024_025.sql +6 -0
- kinto/core/storage/postgresql/migrator.py +98 -0
- kinto/core/storage/postgresql/pool.py +55 -0
- kinto/core/storage/postgresql/schema.sql +143 -0
- kinto/core/storage/testing.py +1857 -0
- kinto/core/storage/utils.py +37 -0
- kinto/core/testing.py +182 -0
- kinto/core/utils.py +553 -0
- kinto/core/views/__init__.py +0 -0
- kinto/core/views/batch.py +163 -0
- kinto/core/views/errors.py +145 -0
- kinto/core/views/heartbeat.py +106 -0
- kinto/core/views/hello.py +69 -0
- kinto/core/views/openapi.py +35 -0
- kinto/core/views/version.py +50 -0
- kinto/events.py +3 -0
- kinto/plugins/__init__.py +0 -0
- kinto/plugins/accounts/__init__.py +94 -0
- kinto/plugins/accounts/authentication.py +63 -0
- kinto/plugins/accounts/scripts.py +61 -0
- kinto/plugins/accounts/utils.py +13 -0
- kinto/plugins/accounts/views.py +136 -0
- kinto/plugins/admin/README.md +3 -0
- kinto/plugins/admin/VERSION +1 -0
- kinto/plugins/admin/__init__.py +40 -0
- kinto/plugins/admin/build/VERSION +1 -0
- kinto/plugins/admin/build/assets/index-CYFwtKtL.css +6 -0
- kinto/plugins/admin/build/assets/index-DJ0m93zA.js +149 -0
- kinto/plugins/admin/build/assets/logo-VBRiKSPX.png +0 -0
- kinto/plugins/admin/build/index.html +18 -0
- kinto/plugins/admin/public/help.html +25 -0
- kinto/plugins/admin/views.py +42 -0
- kinto/plugins/default_bucket/__init__.py +191 -0
- kinto/plugins/flush.py +28 -0
- kinto/plugins/history/__init__.py +65 -0
- kinto/plugins/history/listener.py +181 -0
- kinto/plugins/history/views.py +66 -0
- kinto/plugins/openid/__init__.py +131 -0
- kinto/plugins/openid/utils.py +14 -0
- kinto/plugins/openid/views.py +193 -0
- kinto/plugins/prometheus.py +300 -0
- kinto/plugins/statsd.py +85 -0
- kinto/schema_validation.py +135 -0
- kinto/views/__init__.py +34 -0
- kinto/views/admin.py +195 -0
- kinto/views/buckets.py +45 -0
- kinto/views/collections.py +58 -0
- kinto/views/contribute.py +39 -0
- kinto/views/groups.py +90 -0
- kinto/views/permissions.py +235 -0
- kinto/views/records.py +133 -0
- kinto-23.2.1.dist-info/METADATA +232 -0
- kinto-23.2.1.dist-info/RECORD +142 -0
- kinto-23.2.1.dist-info/WHEEL +5 -0
- kinto-23.2.1.dist-info/entry_points.txt +5 -0
- kinto-23.2.1.dist-info/licenses/LICENSE +13 -0
- kinto-23.2.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,738 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import random
|
|
3
|
+
import re
|
|
4
|
+
import warnings
|
|
5
|
+
from datetime import datetime
|
|
6
|
+
|
|
7
|
+
from dateutil import parser as dateparser
|
|
8
|
+
from dockerflow.logging import get_or_generate_request_id, request_id_context
|
|
9
|
+
from pyramid.events import ApplicationCreated, NewRequest, NewResponse
|
|
10
|
+
from pyramid.exceptions import ConfigurationError
|
|
11
|
+
from pyramid.httpexceptions import (
|
|
12
|
+
HTTPBadRequest,
|
|
13
|
+
HTTPGone,
|
|
14
|
+
HTTPMethodNotAllowed,
|
|
15
|
+
HTTPTemporaryRedirect,
|
|
16
|
+
)
|
|
17
|
+
from pyramid.interfaces import IAuthenticationPolicy
|
|
18
|
+
from pyramid.renderers import JSON as JSONRenderer
|
|
19
|
+
from pyramid.response import Response
|
|
20
|
+
from pyramid.security import NO_PERMISSION_REQUIRED
|
|
21
|
+
from pyramid.settings import asbool, aslist
|
|
22
|
+
from pyramid_multiauth import MultiAuthenticationPolicy, MultiAuthPolicySelected
|
|
23
|
+
|
|
24
|
+
from kinto.core import cache, errors, metrics, permission, storage, utils
|
|
25
|
+
from kinto.core.events import ACTIONS, ResourceChanged, ResourceRead
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
try:
|
|
29
|
+
import newrelic.agent
|
|
30
|
+
except ImportError: # pragma: no cover
|
|
31
|
+
newrelic = None
|
|
32
|
+
try:
|
|
33
|
+
from werkzeug.middleware.profiler import ProfilerMiddleware
|
|
34
|
+
except ImportError: # pragma: no cover
|
|
35
|
+
ProfilerMiddleware = False
|
|
36
|
+
try:
|
|
37
|
+
import sentry_sdk
|
|
38
|
+
from sentry_sdk.integrations.logging import LoggingIntegration
|
|
39
|
+
from sentry_sdk.integrations.pyramid import PyramidIntegration
|
|
40
|
+
from sentry_sdk.integrations.sqlalchemy import SqlalchemyIntegration
|
|
41
|
+
except ImportError: # pragma: no cover
|
|
42
|
+
sentry_sdk = None
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
logger = logging.getLogger(__name__)
|
|
46
|
+
summary_logger = logging.getLogger("request.summary")
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def setup_request_bound_data(config):
|
|
50
|
+
"""Attach custom data on request object, and share it with parent
|
|
51
|
+
requests during batch."""
|
|
52
|
+
|
|
53
|
+
def attach_bound_data(request):
|
|
54
|
+
parent = getattr(request, "parent", None)
|
|
55
|
+
return parent.bound_data if parent else {}
|
|
56
|
+
|
|
57
|
+
config.add_request_method(attach_bound_data, name="bound_data", reify=True)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def setup_json_serializer(config):
|
|
61
|
+
import requests
|
|
62
|
+
import webob
|
|
63
|
+
|
|
64
|
+
# Monkey patch to use rapidjson
|
|
65
|
+
webob.request.json = utils.json
|
|
66
|
+
requests.models.json = utils.json
|
|
67
|
+
|
|
68
|
+
# Override json renderer using rapidjson
|
|
69
|
+
renderer = JSONRenderer(serializer=utils.json_serializer)
|
|
70
|
+
config.add_renderer("ultrajson", renderer) # See `kinto.core.Service`
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def setup_csp_headers(config):
|
|
74
|
+
"""Content-Security-Policy HTTP response header helps reduce XSS risks on
|
|
75
|
+
modern browsers by declaring, which dynamic resources are allowed to load.
|
|
76
|
+
On APIs, we disable everything.
|
|
77
|
+
"""
|
|
78
|
+
disable_all = "default-src 'none'; frame-ancestors 'none'; base-uri 'none'; "
|
|
79
|
+
|
|
80
|
+
def on_new_response(event):
|
|
81
|
+
event.response.headers.setdefault("Content-Security-Policy", disable_all)
|
|
82
|
+
|
|
83
|
+
config.add_subscriber(on_new_response, NewResponse)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def restrict_http_methods_if_readonly(config):
|
|
87
|
+
"""Prevent write operations if server is configured as read-only.
|
|
88
|
+
|
|
89
|
+
This is an additional layer of security on top of the verifications done
|
|
90
|
+
in :method:`kinto.core.Resource.register_service`, in case an installed
|
|
91
|
+
plugin does not take this setting into account in the definition of its
|
|
92
|
+
ad-hoc Pyramid views.
|
|
93
|
+
"""
|
|
94
|
+
settings = config.get_settings()
|
|
95
|
+
|
|
96
|
+
def on_new_request(event):
|
|
97
|
+
if event.request.method.lower() not in ("get", "head", "options"):
|
|
98
|
+
raise HTTPMethodNotAllowed()
|
|
99
|
+
|
|
100
|
+
if asbool(settings["readonly"]):
|
|
101
|
+
config.add_subscriber(on_new_request, NewRequest)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def setup_version_redirection(config):
|
|
105
|
+
"""Add a view which redirects to the current version of the API."""
|
|
106
|
+
settings = config.get_settings()
|
|
107
|
+
redirect_enabled = settings["version_prefix_redirect_enabled"]
|
|
108
|
+
version_prefix_redirection_enabled = asbool(redirect_enabled)
|
|
109
|
+
cache_seconds = int(settings["version_prefix_redirect_ttl_seconds"])
|
|
110
|
+
|
|
111
|
+
route_prefix = config.route_prefix
|
|
112
|
+
config.registry.route_prefix = route_prefix
|
|
113
|
+
|
|
114
|
+
# Redirect to the current version of the API if the prefix isn't used.
|
|
115
|
+
# Do not redirect if kinto.version_prefix_redirect_enabled is set to
|
|
116
|
+
# False.
|
|
117
|
+
if not version_prefix_redirection_enabled:
|
|
118
|
+
return
|
|
119
|
+
|
|
120
|
+
def _redirect_to_version_view(request):
|
|
121
|
+
if request.method.lower() == "options":
|
|
122
|
+
# CORS responses should always have status 200.
|
|
123
|
+
return utils.reapply_cors(request, Response())
|
|
124
|
+
|
|
125
|
+
querystring = request.url[(request.url.rindex(request.path) + len(request.path)) :]
|
|
126
|
+
redirect = f"/{route_prefix}{request.path}{querystring}"
|
|
127
|
+
resp = HTTPTemporaryRedirect(redirect)
|
|
128
|
+
if cache_seconds >= 0:
|
|
129
|
+
resp.cache_expires(cache_seconds)
|
|
130
|
+
raise resp
|
|
131
|
+
|
|
132
|
+
# Disable the route prefix passed by the app.
|
|
133
|
+
config.route_prefix = None
|
|
134
|
+
|
|
135
|
+
config.add_route(name="redirect_to_version", pattern=r"/{path:(?!v[0-9]+)[^\r\n]*}")
|
|
136
|
+
|
|
137
|
+
config.add_view(
|
|
138
|
+
view=_redirect_to_version_view,
|
|
139
|
+
route_name="redirect_to_version",
|
|
140
|
+
permission=NO_PERMISSION_REQUIRED,
|
|
141
|
+
)
|
|
142
|
+
|
|
143
|
+
config.route_prefix = route_prefix
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def setup_authentication(config):
|
|
147
|
+
"""Let pyramid_multiauth manage authentication and authorization
|
|
148
|
+
from configuration.
|
|
149
|
+
"""
|
|
150
|
+
config.include("pyramid_multiauth")
|
|
151
|
+
settings = config.get_settings()
|
|
152
|
+
|
|
153
|
+
policies = aslist(settings["multiauth.policies"])
|
|
154
|
+
if "basicauth" in policies:
|
|
155
|
+
config.include("kinto.core.authentication")
|
|
156
|
+
|
|
157
|
+
# Track policy used, for prefixing user_id and for logging.
|
|
158
|
+
def on_policy_selected(event):
|
|
159
|
+
request = event.request
|
|
160
|
+
# If the authn policy has a name attribute, we use it rather
|
|
161
|
+
# than using the one specified in settings.
|
|
162
|
+
authn_type = getattr(event.policy, "name", event.policy_name.lower())
|
|
163
|
+
request.authn_type = authn_type
|
|
164
|
+
request.selected_userid = event.userid
|
|
165
|
+
# Add authentication info to context.
|
|
166
|
+
request.log_context(uid=event.userid, authn_type=authn_type)
|
|
167
|
+
|
|
168
|
+
config.add_subscriber(on_policy_selected, MultiAuthPolicySelected)
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def setup_backoff(config):
|
|
172
|
+
"""Attach HTTP requests/responses objects.
|
|
173
|
+
|
|
174
|
+
This is useful to attach objects to the request object for easier
|
|
175
|
+
access, and to pre-process responses.
|
|
176
|
+
"""
|
|
177
|
+
|
|
178
|
+
def on_new_response(event):
|
|
179
|
+
# Add backoff in response headers.
|
|
180
|
+
backoff = config.registry.settings["backoff"]
|
|
181
|
+
if backoff is not None:
|
|
182
|
+
backoff_percentage = config.registry.settings["backoff_percentage"]
|
|
183
|
+
if backoff_percentage is not None:
|
|
184
|
+
if random.random() < (float(backoff_percentage) / 100.0):
|
|
185
|
+
event.response.headers["Backoff"] = str(backoff)
|
|
186
|
+
else:
|
|
187
|
+
event.response.headers["Backoff"] = str(backoff)
|
|
188
|
+
|
|
189
|
+
config.add_subscriber(on_new_response, NewResponse)
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def setup_requests_scheme(config):
|
|
193
|
+
"""Force server scheme, host and port at the application level."""
|
|
194
|
+
settings = config.get_settings()
|
|
195
|
+
|
|
196
|
+
http_scheme = settings["http_scheme"]
|
|
197
|
+
http_host = settings["http_host"]
|
|
198
|
+
|
|
199
|
+
def on_new_request(event):
|
|
200
|
+
if http_scheme:
|
|
201
|
+
event.request.scheme = http_scheme
|
|
202
|
+
if http_host:
|
|
203
|
+
event.request.host = http_host
|
|
204
|
+
|
|
205
|
+
if http_scheme or http_host:
|
|
206
|
+
config.add_subscriber(on_new_request, NewRequest)
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def setup_deprecation(config):
|
|
210
|
+
config.add_tween("kinto.core.initialization._end_of_life_tween_factory")
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def _end_of_life_tween_factory(handler, registry):
|
|
214
|
+
"""Pyramid tween to handle service end of life."""
|
|
215
|
+
deprecation_msg = "The service you are trying to connect no longer exists at this location."
|
|
216
|
+
|
|
217
|
+
def eos_tween(request):
|
|
218
|
+
eos_date = registry.settings["eos"]
|
|
219
|
+
eos_url = registry.settings["eos_url"]
|
|
220
|
+
eos_message = registry.settings["eos_message"]
|
|
221
|
+
if not eos_date:
|
|
222
|
+
return handler(request)
|
|
223
|
+
|
|
224
|
+
eos_date = dateparser.parse(eos_date)
|
|
225
|
+
if eos_date > datetime.now():
|
|
226
|
+
code = "soft-eol"
|
|
227
|
+
request.response = handler(request)
|
|
228
|
+
else:
|
|
229
|
+
code = "hard-eol"
|
|
230
|
+
request.response = errors.http_error(
|
|
231
|
+
HTTPGone(),
|
|
232
|
+
errno=errors.ERRORS.SERVICE_DEPRECATED,
|
|
233
|
+
message=deprecation_msg,
|
|
234
|
+
)
|
|
235
|
+
|
|
236
|
+
errors.send_alert(request, eos_message, url=eos_url, code=code)
|
|
237
|
+
return utils.reapply_cors(request, request.response)
|
|
238
|
+
|
|
239
|
+
return eos_tween
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def setup_storage(config):
|
|
243
|
+
settings = config.get_settings()
|
|
244
|
+
|
|
245
|
+
# Id generators by resource name.
|
|
246
|
+
config.registry.id_generators = {}
|
|
247
|
+
for key, value in settings.items():
|
|
248
|
+
m = re.match(r"^([^_]*)_?id_generator", key)
|
|
249
|
+
if m is None:
|
|
250
|
+
continue
|
|
251
|
+
resource_name = m.group(1)
|
|
252
|
+
id_generator = config.maybe_dotted(value)
|
|
253
|
+
config.registry.id_generators[resource_name] = id_generator()
|
|
254
|
+
|
|
255
|
+
storage_mod = settings["storage_backend"]
|
|
256
|
+
if not storage_mod:
|
|
257
|
+
return
|
|
258
|
+
|
|
259
|
+
storage_mod = config.maybe_dotted(storage_mod)
|
|
260
|
+
backend = storage_mod.load_from_config(config)
|
|
261
|
+
if not isinstance(backend, storage.StorageBase):
|
|
262
|
+
raise ConfigurationError(f"Invalid storage backend: {backend}")
|
|
263
|
+
config.registry.storage = backend
|
|
264
|
+
|
|
265
|
+
heartbeat = storage.heartbeat(backend)
|
|
266
|
+
config.registry.heartbeats["storage"] = heartbeat
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def setup_permission(config):
|
|
270
|
+
settings = config.get_settings()
|
|
271
|
+
permission_mod = settings["permission_backend"]
|
|
272
|
+
if not permission_mod:
|
|
273
|
+
return
|
|
274
|
+
|
|
275
|
+
permission_mod = config.maybe_dotted(permission_mod)
|
|
276
|
+
backend = permission_mod.load_from_config(config)
|
|
277
|
+
if not isinstance(backend, permission.PermissionBase):
|
|
278
|
+
raise ConfigurationError(f"Invalid permission backend: {backend}")
|
|
279
|
+
config.registry.permission = backend
|
|
280
|
+
|
|
281
|
+
heartbeat = permission.heartbeat(backend)
|
|
282
|
+
config.registry.heartbeats["permission"] = heartbeat
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def setup_cache(config):
|
|
286
|
+
settings = config.get_settings()
|
|
287
|
+
cache_mod = settings["cache_backend"]
|
|
288
|
+
if not cache_mod:
|
|
289
|
+
return
|
|
290
|
+
|
|
291
|
+
cache_mod = config.maybe_dotted(cache_mod)
|
|
292
|
+
backend = cache_mod.load_from_config(config)
|
|
293
|
+
if not isinstance(backend, cache.CacheBase):
|
|
294
|
+
raise ConfigurationError(f"Invalid cache backend: {backend}")
|
|
295
|
+
config.registry.cache = backend
|
|
296
|
+
|
|
297
|
+
heartbeat = cache.heartbeat(backend)
|
|
298
|
+
config.registry.heartbeats["cache"] = heartbeat
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
def setup_sentry(config):
|
|
302
|
+
settings = config.get_settings()
|
|
303
|
+
|
|
304
|
+
# Note: SENTRY_DSN and SENTRY_ENV env variables will override
|
|
305
|
+
# .ini values thanks to `load_default_settings()`.
|
|
306
|
+
|
|
307
|
+
dsn = settings["sentry_dsn"]
|
|
308
|
+
if dsn: # pragma: no cover
|
|
309
|
+
env_options = {}
|
|
310
|
+
env = settings["sentry_env"]
|
|
311
|
+
if env:
|
|
312
|
+
env_options["environment"] = env
|
|
313
|
+
|
|
314
|
+
breadcrumbs_level = settings["sentry_breadcrumbs_min_level"]
|
|
315
|
+
events_level = settings["sentry_events_min_level"]
|
|
316
|
+
sentry_sdk.init(
|
|
317
|
+
dsn,
|
|
318
|
+
integrations=[
|
|
319
|
+
PyramidIntegration(),
|
|
320
|
+
SqlalchemyIntegration(),
|
|
321
|
+
LoggingIntegration(
|
|
322
|
+
# Logs to be captured as breadcrumbs (debug and above by default)
|
|
323
|
+
level=breadcrumbs_level,
|
|
324
|
+
# Logs to be catpured as events (warning and above by default)
|
|
325
|
+
event_level=events_level,
|
|
326
|
+
),
|
|
327
|
+
],
|
|
328
|
+
**env_options,
|
|
329
|
+
)
|
|
330
|
+
|
|
331
|
+
def on_app_created(event):
|
|
332
|
+
msg = "Running {project_name} {project_version}.".format_map(settings)
|
|
333
|
+
sentry_sdk.capture_message(msg, "info")
|
|
334
|
+
|
|
335
|
+
config.add_subscriber(on_app_created, ApplicationCreated)
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
def install_middlewares(app, settings):
|
|
339
|
+
"Install a set of middlewares defined in the ini file on the given app."
|
|
340
|
+
# Setup new-relic.
|
|
341
|
+
if settings.get("newrelic_config"):
|
|
342
|
+
ini_file = settings["newrelic_config"]
|
|
343
|
+
env = settings["newrelic_env"]
|
|
344
|
+
newrelic.agent.initialize(ini_file, env)
|
|
345
|
+
app = newrelic.agent.WSGIApplicationWrapper(app)
|
|
346
|
+
|
|
347
|
+
# Adds the Werkzeug profiler.
|
|
348
|
+
if asbool(settings.get("profiler_enabled")):
|
|
349
|
+
profile_dir = settings["profiler_dir"]
|
|
350
|
+
app = ProfilerMiddleware(app, profile_dir=profile_dir, restrictions=("*kinto.core*"))
|
|
351
|
+
|
|
352
|
+
return app
|
|
353
|
+
|
|
354
|
+
|
|
355
|
+
def setup_logging(config):
|
|
356
|
+
"""Setup structured logging, and emit `request.summary` event on each
|
|
357
|
+
request, as recommended by Mozilla Services standard:
|
|
358
|
+
|
|
359
|
+
* https://mana.mozilla.org/wiki/display/CLOUDSERVICES/Logging+Standard
|
|
360
|
+
* http://12factor.net/logs
|
|
361
|
+
"""
|
|
362
|
+
|
|
363
|
+
def on_new_request(event):
|
|
364
|
+
request = event.request
|
|
365
|
+
# Save the time the request was received by the server.
|
|
366
|
+
event.request._received_at = utils.msec_time()
|
|
367
|
+
|
|
368
|
+
try:
|
|
369
|
+
# Pyramid fails if the URL contains invalid UTF-8 characters.
|
|
370
|
+
request_path = event.request.path
|
|
371
|
+
except UnicodeDecodeError:
|
|
372
|
+
raise errors.http_error(
|
|
373
|
+
HTTPBadRequest(),
|
|
374
|
+
errno=errors.ERRORS.INVALID_PARAMETERS,
|
|
375
|
+
message="Invalid URL path.",
|
|
376
|
+
)
|
|
377
|
+
|
|
378
|
+
rid = get_or_generate_request_id(headers=request.headers)
|
|
379
|
+
request_id_context.set(rid)
|
|
380
|
+
|
|
381
|
+
request.log_context(
|
|
382
|
+
agent=request.headers.get("User-Agent"),
|
|
383
|
+
path=request_path,
|
|
384
|
+
method=request.method,
|
|
385
|
+
lang=request.headers.get("Accept-Language"),
|
|
386
|
+
rid=rid,
|
|
387
|
+
errno=0,
|
|
388
|
+
)
|
|
389
|
+
qs = dict(errors.request_GET(request))
|
|
390
|
+
if qs:
|
|
391
|
+
request.log_context(querystring=qs)
|
|
392
|
+
|
|
393
|
+
if summary_logger.level == logging.DEBUG:
|
|
394
|
+
request.log_context(headers=dict(request.headers), body=request.body)
|
|
395
|
+
|
|
396
|
+
config.add_subscriber(on_new_request, NewRequest)
|
|
397
|
+
|
|
398
|
+
def on_new_response(event):
|
|
399
|
+
response = event.response
|
|
400
|
+
request = event.request
|
|
401
|
+
|
|
402
|
+
# Compute the request processing time in msec (-1 if unknown)
|
|
403
|
+
current = utils.msec_time()
|
|
404
|
+
duration = current - getattr(request, "_received_at", current - 1)
|
|
405
|
+
isotimestamp = datetime.fromtimestamp(current / 1000).isoformat()
|
|
406
|
+
|
|
407
|
+
# Bind infos for request summary logger.
|
|
408
|
+
request.log_context(time=isotimestamp, code=response.status_code, t=duration)
|
|
409
|
+
|
|
410
|
+
if summary_logger.level == logging.DEBUG:
|
|
411
|
+
request.log_context(response=dict(headers=dict(response.headers), body=response.body))
|
|
412
|
+
|
|
413
|
+
try:
|
|
414
|
+
# If error response, bind errno.
|
|
415
|
+
request.log_context(errno=response.errno)
|
|
416
|
+
except AttributeError:
|
|
417
|
+
pass
|
|
418
|
+
|
|
419
|
+
if not hasattr(request, "parent"):
|
|
420
|
+
# Ouput application request summary.
|
|
421
|
+
summary_logger.info("", extra=request.log_context())
|
|
422
|
+
|
|
423
|
+
config.add_subscriber(on_new_response, NewResponse)
|
|
424
|
+
|
|
425
|
+
|
|
426
|
+
def setup_metrics(config):
|
|
427
|
+
settings = config.get_settings()
|
|
428
|
+
|
|
429
|
+
# Register a no-op metrics service by default.
|
|
430
|
+
config.registry.registerUtility(metrics.NoOpMetricsService(), metrics.IMetricsService)
|
|
431
|
+
|
|
432
|
+
# This does not fully respect the Pyramid/ZCA patterns, but the rest of Kinto uses
|
|
433
|
+
# `registry.storage`, `registry.cache`, etc. Consistency seems more important.
|
|
434
|
+
config.registry.__class__.metrics = property(
|
|
435
|
+
lambda reg: reg.queryUtility(metrics.IMetricsService)
|
|
436
|
+
)
|
|
437
|
+
|
|
438
|
+
def deprecated_registry(self):
|
|
439
|
+
warnings.warn(
|
|
440
|
+
"``config.registry.statsd`` is now deprecated. Use ``config.registry.metrics`` instead.",
|
|
441
|
+
DeprecationWarning,
|
|
442
|
+
)
|
|
443
|
+
return self.metrics
|
|
444
|
+
|
|
445
|
+
config.registry.__class__.statsd = property(deprecated_registry)
|
|
446
|
+
|
|
447
|
+
def on_app_created(event):
|
|
448
|
+
config = event.app
|
|
449
|
+
metrics_service = config.registry.metrics
|
|
450
|
+
|
|
451
|
+
metrics.watch_execution_time(metrics_service, config.registry.cache, prefix="backend")
|
|
452
|
+
metrics.watch_execution_time(metrics_service, config.registry.storage, prefix="backend")
|
|
453
|
+
metrics.watch_execution_time(metrics_service, config.registry.permission, prefix="backend")
|
|
454
|
+
|
|
455
|
+
policy = config.registry.queryUtility(IAuthenticationPolicy)
|
|
456
|
+
if isinstance(policy, MultiAuthenticationPolicy):
|
|
457
|
+
for name, subpolicy in policy.get_policies():
|
|
458
|
+
metrics.watch_execution_time(
|
|
459
|
+
metrics_service, subpolicy, prefix="authentication", classname=name
|
|
460
|
+
)
|
|
461
|
+
else:
|
|
462
|
+
metrics.watch_execution_time(metrics_service, policy, prefix="authentication")
|
|
463
|
+
|
|
464
|
+
# Set cache metrics backend
|
|
465
|
+
cache_backend = config.registry.cache
|
|
466
|
+
if isinstance(cache_backend, cache.CacheBase):
|
|
467
|
+
cache_backend.set_metrics_backend(metrics_service)
|
|
468
|
+
|
|
469
|
+
config.add_subscriber(on_app_created, ApplicationCreated)
|
|
470
|
+
|
|
471
|
+
def on_new_response(event):
|
|
472
|
+
request = event.request
|
|
473
|
+
metrics_service = config.registry.metrics
|
|
474
|
+
|
|
475
|
+
# Count unique users.
|
|
476
|
+
user_id = request.prefixed_userid
|
|
477
|
+
if user_id:
|
|
478
|
+
auth, user_id = user_id.split(":", 1)
|
|
479
|
+
# Get rid of colons in metric packet (see #1282 and #3571).
|
|
480
|
+
user_id = user_id.replace(":", ".")
|
|
481
|
+
metrics_service.count("users", unique=[("auth", auth), ("userid", user_id)])
|
|
482
|
+
|
|
483
|
+
status = event.response.status_code
|
|
484
|
+
|
|
485
|
+
if status >= 400:
|
|
486
|
+
# Prevent random values of 404 responses to become label values.
|
|
487
|
+
request_matchdict = {}
|
|
488
|
+
else:
|
|
489
|
+
request_matchdict = dict(request.matchdict or {})
|
|
490
|
+
|
|
491
|
+
# Add extra labels to metrics, based on fields extracted from the request matchdict.
|
|
492
|
+
metrics_matchdict_fields = aslist(settings["metrics_matchdict_fields"])
|
|
493
|
+
# Turn the `id` field of object endpoints into `{resource}_id` (eg. `mushroom_id`, `bucket_id`)
|
|
494
|
+
enhanced_matchdict = request_matchdict
|
|
495
|
+
try:
|
|
496
|
+
enhanced_matchdict[request.current_resource_name + "_id"] = enhanced_matchdict.get(
|
|
497
|
+
"id", ""
|
|
498
|
+
)
|
|
499
|
+
except AttributeError:
|
|
500
|
+
# Not on a resource.
|
|
501
|
+
pass
|
|
502
|
+
metrics_matchdict_labels = [
|
|
503
|
+
(field, enhanced_matchdict.get(field, "")) for field in metrics_matchdict_fields
|
|
504
|
+
]
|
|
505
|
+
|
|
506
|
+
service = request.current_service
|
|
507
|
+
if service:
|
|
508
|
+
# Use the service name as endpoint if available.
|
|
509
|
+
endpoint = service.name
|
|
510
|
+
elif route := request.matched_route:
|
|
511
|
+
# Use the route name as endpoint if we're not on a Cornice service.
|
|
512
|
+
endpoint = route.name
|
|
513
|
+
else:
|
|
514
|
+
endpoint = (
|
|
515
|
+
"unnamed" if status != 404 else "unknown"
|
|
516
|
+
) # Do not multiply cardinality for unknown endpoints.
|
|
517
|
+
|
|
518
|
+
request_labels = [
|
|
519
|
+
("method", request.method.lower()),
|
|
520
|
+
("endpoint", endpoint),
|
|
521
|
+
] + metrics_matchdict_labels
|
|
522
|
+
|
|
523
|
+
# Count served requests.
|
|
524
|
+
metrics_service.count("request_summary", unique=request_labels + [("status", str(status))])
|
|
525
|
+
|
|
526
|
+
try:
|
|
527
|
+
current = utils.msec_time()
|
|
528
|
+
duration = (current - request._received_at) / 1000
|
|
529
|
+
metrics_service.timer(
|
|
530
|
+
"request_duration_seconds",
|
|
531
|
+
value=duration,
|
|
532
|
+
labels=request_labels,
|
|
533
|
+
)
|
|
534
|
+
except AttributeError: # pragma: no cover
|
|
535
|
+
# Logging was not setup in this Kinto app (unlikely but possible)
|
|
536
|
+
pass
|
|
537
|
+
|
|
538
|
+
# Observe response size.
|
|
539
|
+
metrics_service.observe(
|
|
540
|
+
"request_size", len(event.response.body or b""), labels=request_labels
|
|
541
|
+
)
|
|
542
|
+
|
|
543
|
+
# Count authentication verifications.
|
|
544
|
+
try:
|
|
545
|
+
metrics_service.count("authentication", unique=[("type", request.authn_type)])
|
|
546
|
+
except AttributeError:
|
|
547
|
+
# Not authenticated
|
|
548
|
+
pass
|
|
549
|
+
|
|
550
|
+
config.add_subscriber(on_new_response, NewResponse)
|
|
551
|
+
|
|
552
|
+
|
|
553
|
+
class EventActionFilter:
|
|
554
|
+
def __init__(self, actions, config):
|
|
555
|
+
actions = ACTIONS.from_string_list(actions)
|
|
556
|
+
self.actions = [action.value for action in actions]
|
|
557
|
+
|
|
558
|
+
def phash(self):
|
|
559
|
+
return f"for_actions = {','.join(self.actions)}"
|
|
560
|
+
|
|
561
|
+
def __call__(self, event):
|
|
562
|
+
action = event.payload.get("action")
|
|
563
|
+
return not action or action in self.actions
|
|
564
|
+
|
|
565
|
+
|
|
566
|
+
class EventResourceFilter:
|
|
567
|
+
def __init__(self, resources, config):
|
|
568
|
+
self.resources = resources
|
|
569
|
+
|
|
570
|
+
def phash(self):
|
|
571
|
+
return f"for_resources = {','.join(self.resources)}"
|
|
572
|
+
|
|
573
|
+
def __call__(self, event):
|
|
574
|
+
resource = event.payload.get("resource_name")
|
|
575
|
+
return not resource or not self.resources or resource in self.resources
|
|
576
|
+
|
|
577
|
+
|
|
578
|
+
def setup_listeners(config):
|
|
579
|
+
# Register basic subscriber predicates, to filter events.
|
|
580
|
+
config.add_subscriber_predicate("for_actions", EventActionFilter)
|
|
581
|
+
config.add_subscriber_predicate("for_resources", EventResourceFilter)
|
|
582
|
+
|
|
583
|
+
write_actions = (ACTIONS.CREATE, ACTIONS.UPDATE, ACTIONS.DELETE)
|
|
584
|
+
settings = config.get_settings()
|
|
585
|
+
settings_prefix = settings.get("settings_prefix", "")
|
|
586
|
+
listeners = aslist(settings["event_listeners"])
|
|
587
|
+
|
|
588
|
+
for name in listeners:
|
|
589
|
+
logger.info(f"Setting up '{name}' listener")
|
|
590
|
+
prefix = f"event_listeners.{name}."
|
|
591
|
+
|
|
592
|
+
try:
|
|
593
|
+
listener_mod = config.maybe_dotted(name)
|
|
594
|
+
prefix = f"event_listeners.{name.split('.')[-1]}."
|
|
595
|
+
listener = listener_mod.load_from_config(config, prefix)
|
|
596
|
+
except (ImportError, AttributeError):
|
|
597
|
+
module_setting = prefix + "use"
|
|
598
|
+
# Read from ENV or settings.
|
|
599
|
+
module_value = utils.read_env(
|
|
600
|
+
f"{settings_prefix}.{module_setting}", settings.get(module_setting)
|
|
601
|
+
)
|
|
602
|
+
listener_mod = config.maybe_dotted(module_value)
|
|
603
|
+
listener = listener_mod.load_from_config(config, prefix)
|
|
604
|
+
|
|
605
|
+
wrapped_listener = metrics.listener_with_timer(
|
|
606
|
+
config, f"listeners.{name}", listener.__call__
|
|
607
|
+
)
|
|
608
|
+
|
|
609
|
+
# Optional filter by event action.
|
|
610
|
+
actions_setting = prefix + "actions"
|
|
611
|
+
# Read from ENV or settings.
|
|
612
|
+
actions_value = utils.read_env(
|
|
613
|
+
f"{settings_prefix}.{actions_setting}", settings.get(actions_setting, "")
|
|
614
|
+
)
|
|
615
|
+
actions = aslist(actions_value)
|
|
616
|
+
if len(actions) > 0:
|
|
617
|
+
actions = ACTIONS.from_string_list(actions)
|
|
618
|
+
else:
|
|
619
|
+
actions = write_actions
|
|
620
|
+
|
|
621
|
+
# Optional filter by event resource name.
|
|
622
|
+
resource_setting = prefix + "resources"
|
|
623
|
+
# Read from ENV or settings.
|
|
624
|
+
resource_value = utils.read_env(
|
|
625
|
+
f"{settings_prefix}.{resource_setting}", settings.get(resource_setting, "")
|
|
626
|
+
)
|
|
627
|
+
resource_names = aslist(resource_value)
|
|
628
|
+
|
|
629
|
+
# Pyramid event predicates.
|
|
630
|
+
options = dict(for_actions=actions, for_resources=resource_names)
|
|
631
|
+
|
|
632
|
+
if ACTIONS.READ in actions:
|
|
633
|
+
config.add_subscriber(wrapped_listener, ResourceRead, **options)
|
|
634
|
+
actions = [a for a in actions if a != ACTIONS.READ]
|
|
635
|
+
|
|
636
|
+
if len(actions) > 0:
|
|
637
|
+
config.add_subscriber(wrapped_listener, ResourceChanged, **options)
|
|
638
|
+
|
|
639
|
+
|
|
640
|
+
def load_default_settings(config, default_settings):
|
|
641
|
+
"""Read settings provided in Paste ini file, set default values and
|
|
642
|
+
replace if defined as environment variable.
|
|
643
|
+
"""
|
|
644
|
+
settings = config.get_settings()
|
|
645
|
+
|
|
646
|
+
settings_prefix = settings["settings_prefix"]
|
|
647
|
+
|
|
648
|
+
def _prefixed_keys(key):
|
|
649
|
+
unprefixed = key
|
|
650
|
+
if key.startswith(settings_prefix + "."):
|
|
651
|
+
unprefixed = key.split(".", 1)[1]
|
|
652
|
+
project_prefix = f"{settings_prefix}.{unprefixed}"
|
|
653
|
+
return unprefixed, project_prefix
|
|
654
|
+
|
|
655
|
+
# Fill settings with default values if not defined.
|
|
656
|
+
for key, default_value in sorted(default_settings.items()):
|
|
657
|
+
unprefixed, project_prefix = keys = _prefixed_keys(key)
|
|
658
|
+
is_defined = len(set(settings.keys()).intersection(set(keys))) > 0
|
|
659
|
+
if not is_defined:
|
|
660
|
+
settings[unprefixed] = default_value
|
|
661
|
+
|
|
662
|
+
for key, value in sorted(settings.items()):
|
|
663
|
+
value = utils.native_value(value)
|
|
664
|
+
unprefixed, project_prefix = keys = _prefixed_keys(key)
|
|
665
|
+
|
|
666
|
+
# Fail if not only one is defined.
|
|
667
|
+
defined = set(settings.keys()).intersection(set(keys))
|
|
668
|
+
distinct_values = set([str(settings[d]) for d in defined])
|
|
669
|
+
|
|
670
|
+
if len(defined) > 1 and len(distinct_values) > 1:
|
|
671
|
+
names = "', '".join(defined)
|
|
672
|
+
raise ValueError(f"Settings '{names}' are in conflict.")
|
|
673
|
+
|
|
674
|
+
# Override settings from OS env values.
|
|
675
|
+
# e.g. HTTP_PORT, READINGLIST_HTTP_PORT, KINTO_HTTP_PORT
|
|
676
|
+
from_env = utils.read_env(unprefixed, value)
|
|
677
|
+
from_env = utils.read_env(project_prefix, from_env)
|
|
678
|
+
|
|
679
|
+
settings[unprefixed] = from_env
|
|
680
|
+
|
|
681
|
+
config.add_settings(settings)
|
|
682
|
+
|
|
683
|
+
|
|
684
|
+
def initialize(config, version=None, settings_prefix="", default_settings=None):
|
|
685
|
+
"""Initialize kinto.core with the given configuration, version and project
|
|
686
|
+
name.
|
|
687
|
+
|
|
688
|
+
This will basically include kinto.core in Pyramid and set route prefix
|
|
689
|
+
based on the specified version.
|
|
690
|
+
|
|
691
|
+
:param config: Pyramid configuration
|
|
692
|
+
:type config: :class:`~pyramid:pyramid.config.Configurator`
|
|
693
|
+
:param str version: Current project version (e.g. '0.0.1') if not defined
|
|
694
|
+
in application settings.
|
|
695
|
+
:param str settings_prefix: Project name if not defined
|
|
696
|
+
in application settings.
|
|
697
|
+
:param dict default_settings: Override kinto.core default settings values.
|
|
698
|
+
"""
|
|
699
|
+
from kinto.core import DEFAULT_SETTINGS
|
|
700
|
+
|
|
701
|
+
settings = config.get_settings()
|
|
702
|
+
|
|
703
|
+
settings_prefix = (
|
|
704
|
+
settings.pop("kinto.settings_prefix", settings.get("settings_prefix")) or settings_prefix
|
|
705
|
+
)
|
|
706
|
+
settings["settings_prefix"] = settings_prefix
|
|
707
|
+
if not settings_prefix:
|
|
708
|
+
warnings.warn("No value specified for `settings_prefix`")
|
|
709
|
+
|
|
710
|
+
kinto_core_defaults = {**DEFAULT_SETTINGS}
|
|
711
|
+
|
|
712
|
+
if default_settings:
|
|
713
|
+
kinto_core_defaults.update(default_settings)
|
|
714
|
+
|
|
715
|
+
load_default_settings(config, kinto_core_defaults)
|
|
716
|
+
|
|
717
|
+
http_scheme = settings["http_scheme"]
|
|
718
|
+
if http_scheme != "https":
|
|
719
|
+
warnings.warn("HTTPS is not enabled")
|
|
720
|
+
|
|
721
|
+
# Override project version from settings.
|
|
722
|
+
project_version = settings.get("project_version") or version
|
|
723
|
+
if not project_version:
|
|
724
|
+
error_msg = f"Invalid project version: {project_version}"
|
|
725
|
+
raise ConfigurationError(error_msg)
|
|
726
|
+
settings["project_version"] = project_version = str(project_version)
|
|
727
|
+
|
|
728
|
+
# HTTP API version.
|
|
729
|
+
http_api_version = settings.get("http_api_version")
|
|
730
|
+
if http_api_version is None:
|
|
731
|
+
# The API version is derivated from the module version if not provided.
|
|
732
|
+
http_api_version = ".".join(project_version.split(".")[0:2])
|
|
733
|
+
settings["http_api_version"] = http_api_version = str(http_api_version)
|
|
734
|
+
api_version = f"v{http_api_version.split('.')[0]}"
|
|
735
|
+
|
|
736
|
+
# Include kinto.core views with the correct api version prefix.
|
|
737
|
+
config.include("kinto.core", route_prefix=api_version)
|
|
738
|
+
config.route_prefix = api_version
|