kinto 23.2.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (142) hide show
  1. kinto/__init__.py +92 -0
  2. kinto/__main__.py +249 -0
  3. kinto/authorization.py +134 -0
  4. kinto/config/__init__.py +94 -0
  5. kinto/config/kinto.tpl +270 -0
  6. kinto/contribute.json +27 -0
  7. kinto/core/__init__.py +246 -0
  8. kinto/core/authentication.py +48 -0
  9. kinto/core/authorization.py +311 -0
  10. kinto/core/cache/__init__.py +131 -0
  11. kinto/core/cache/memcached.py +112 -0
  12. kinto/core/cache/memory.py +104 -0
  13. kinto/core/cache/postgresql/__init__.py +178 -0
  14. kinto/core/cache/postgresql/schema.sql +23 -0
  15. kinto/core/cache/testing.py +208 -0
  16. kinto/core/cornice/__init__.py +93 -0
  17. kinto/core/cornice/cors.py +144 -0
  18. kinto/core/cornice/errors.py +40 -0
  19. kinto/core/cornice/pyramidhook.py +373 -0
  20. kinto/core/cornice/renderer.py +89 -0
  21. kinto/core/cornice/resource.py +205 -0
  22. kinto/core/cornice/service.py +641 -0
  23. kinto/core/cornice/util.py +138 -0
  24. kinto/core/cornice/validators/__init__.py +94 -0
  25. kinto/core/cornice/validators/_colander.py +142 -0
  26. kinto/core/cornice/validators/_marshmallow.py +182 -0
  27. kinto/core/cornice_swagger/__init__.py +92 -0
  28. kinto/core/cornice_swagger/converters/__init__.py +21 -0
  29. kinto/core/cornice_swagger/converters/exceptions.py +6 -0
  30. kinto/core/cornice_swagger/converters/parameters.py +90 -0
  31. kinto/core/cornice_swagger/converters/schema.py +249 -0
  32. kinto/core/cornice_swagger/swagger.py +725 -0
  33. kinto/core/cornice_swagger/templates/index.html +73 -0
  34. kinto/core/cornice_swagger/templates/index_script_template.html +21 -0
  35. kinto/core/cornice_swagger/util.py +42 -0
  36. kinto/core/cornice_swagger/views.py +78 -0
  37. kinto/core/decorators.py +74 -0
  38. kinto/core/errors.py +216 -0
  39. kinto/core/events.py +301 -0
  40. kinto/core/initialization.py +738 -0
  41. kinto/core/listeners/__init__.py +9 -0
  42. kinto/core/metrics.py +94 -0
  43. kinto/core/openapi.py +115 -0
  44. kinto/core/permission/__init__.py +202 -0
  45. kinto/core/permission/memory.py +167 -0
  46. kinto/core/permission/postgresql/__init__.py +489 -0
  47. kinto/core/permission/postgresql/migrations/migration_001_002.sql +18 -0
  48. kinto/core/permission/postgresql/schema.sql +41 -0
  49. kinto/core/permission/testing.py +487 -0
  50. kinto/core/resource/__init__.py +1311 -0
  51. kinto/core/resource/model.py +412 -0
  52. kinto/core/resource/schema.py +502 -0
  53. kinto/core/resource/viewset.py +230 -0
  54. kinto/core/schema.py +119 -0
  55. kinto/core/scripts.py +50 -0
  56. kinto/core/statsd.py +1 -0
  57. kinto/core/storage/__init__.py +436 -0
  58. kinto/core/storage/exceptions.py +53 -0
  59. kinto/core/storage/generators.py +58 -0
  60. kinto/core/storage/memory.py +651 -0
  61. kinto/core/storage/postgresql/__init__.py +1131 -0
  62. kinto/core/storage/postgresql/client.py +120 -0
  63. kinto/core/storage/postgresql/migrations/migration_001_002.sql +10 -0
  64. kinto/core/storage/postgresql/migrations/migration_002_003.sql +33 -0
  65. kinto/core/storage/postgresql/migrations/migration_003_004.sql +18 -0
  66. kinto/core/storage/postgresql/migrations/migration_004_005.sql +20 -0
  67. kinto/core/storage/postgresql/migrations/migration_005_006.sql +11 -0
  68. kinto/core/storage/postgresql/migrations/migration_006_007.sql +74 -0
  69. kinto/core/storage/postgresql/migrations/migration_007_008.sql +66 -0
  70. kinto/core/storage/postgresql/migrations/migration_008_009.sql +41 -0
  71. kinto/core/storage/postgresql/migrations/migration_009_010.sql +98 -0
  72. kinto/core/storage/postgresql/migrations/migration_010_011.sql +14 -0
  73. kinto/core/storage/postgresql/migrations/migration_011_012.sql +9 -0
  74. kinto/core/storage/postgresql/migrations/migration_012_013.sql +71 -0
  75. kinto/core/storage/postgresql/migrations/migration_013_014.sql +14 -0
  76. kinto/core/storage/postgresql/migrations/migration_014_015.sql +95 -0
  77. kinto/core/storage/postgresql/migrations/migration_015_016.sql +4 -0
  78. kinto/core/storage/postgresql/migrations/migration_016_017.sql +81 -0
  79. kinto/core/storage/postgresql/migrations/migration_017_018.sql +25 -0
  80. kinto/core/storage/postgresql/migrations/migration_018_019.sql +8 -0
  81. kinto/core/storage/postgresql/migrations/migration_019_020.sql +7 -0
  82. kinto/core/storage/postgresql/migrations/migration_020_021.sql +68 -0
  83. kinto/core/storage/postgresql/migrations/migration_021_022.sql +62 -0
  84. kinto/core/storage/postgresql/migrations/migration_022_023.sql +5 -0
  85. kinto/core/storage/postgresql/migrations/migration_023_024.sql +6 -0
  86. kinto/core/storage/postgresql/migrations/migration_024_025.sql +6 -0
  87. kinto/core/storage/postgresql/migrator.py +98 -0
  88. kinto/core/storage/postgresql/pool.py +55 -0
  89. kinto/core/storage/postgresql/schema.sql +143 -0
  90. kinto/core/storage/testing.py +1857 -0
  91. kinto/core/storage/utils.py +37 -0
  92. kinto/core/testing.py +182 -0
  93. kinto/core/utils.py +553 -0
  94. kinto/core/views/__init__.py +0 -0
  95. kinto/core/views/batch.py +163 -0
  96. kinto/core/views/errors.py +145 -0
  97. kinto/core/views/heartbeat.py +106 -0
  98. kinto/core/views/hello.py +69 -0
  99. kinto/core/views/openapi.py +35 -0
  100. kinto/core/views/version.py +50 -0
  101. kinto/events.py +3 -0
  102. kinto/plugins/__init__.py +0 -0
  103. kinto/plugins/accounts/__init__.py +94 -0
  104. kinto/plugins/accounts/authentication.py +63 -0
  105. kinto/plugins/accounts/scripts.py +61 -0
  106. kinto/plugins/accounts/utils.py +13 -0
  107. kinto/plugins/accounts/views.py +136 -0
  108. kinto/plugins/admin/README.md +3 -0
  109. kinto/plugins/admin/VERSION +1 -0
  110. kinto/plugins/admin/__init__.py +40 -0
  111. kinto/plugins/admin/build/VERSION +1 -0
  112. kinto/plugins/admin/build/assets/index-CYFwtKtL.css +6 -0
  113. kinto/plugins/admin/build/assets/index-DJ0m93zA.js +149 -0
  114. kinto/plugins/admin/build/assets/logo-VBRiKSPX.png +0 -0
  115. kinto/plugins/admin/build/index.html +18 -0
  116. kinto/plugins/admin/public/help.html +25 -0
  117. kinto/plugins/admin/views.py +42 -0
  118. kinto/plugins/default_bucket/__init__.py +191 -0
  119. kinto/plugins/flush.py +28 -0
  120. kinto/plugins/history/__init__.py +65 -0
  121. kinto/plugins/history/listener.py +181 -0
  122. kinto/plugins/history/views.py +66 -0
  123. kinto/plugins/openid/__init__.py +131 -0
  124. kinto/plugins/openid/utils.py +14 -0
  125. kinto/plugins/openid/views.py +193 -0
  126. kinto/plugins/prometheus.py +300 -0
  127. kinto/plugins/statsd.py +85 -0
  128. kinto/schema_validation.py +135 -0
  129. kinto/views/__init__.py +34 -0
  130. kinto/views/admin.py +195 -0
  131. kinto/views/buckets.py +45 -0
  132. kinto/views/collections.py +58 -0
  133. kinto/views/contribute.py +39 -0
  134. kinto/views/groups.py +90 -0
  135. kinto/views/permissions.py +235 -0
  136. kinto/views/records.py +133 -0
  137. kinto-23.2.1.dist-info/METADATA +232 -0
  138. kinto-23.2.1.dist-info/RECORD +142 -0
  139. kinto-23.2.1.dist-info/WHEEL +5 -0
  140. kinto-23.2.1.dist-info/entry_points.txt +5 -0
  141. kinto-23.2.1.dist-info/licenses/LICENSE +13 -0
  142. kinto-23.2.1.dist-info/top_level.txt +1 -0
@@ -0,0 +1,738 @@
1
+ import logging
2
+ import random
3
+ import re
4
+ import warnings
5
+ from datetime import datetime
6
+
7
+ from dateutil import parser as dateparser
8
+ from dockerflow.logging import get_or_generate_request_id, request_id_context
9
+ from pyramid.events import ApplicationCreated, NewRequest, NewResponse
10
+ from pyramid.exceptions import ConfigurationError
11
+ from pyramid.httpexceptions import (
12
+ HTTPBadRequest,
13
+ HTTPGone,
14
+ HTTPMethodNotAllowed,
15
+ HTTPTemporaryRedirect,
16
+ )
17
+ from pyramid.interfaces import IAuthenticationPolicy
18
+ from pyramid.renderers import JSON as JSONRenderer
19
+ from pyramid.response import Response
20
+ from pyramid.security import NO_PERMISSION_REQUIRED
21
+ from pyramid.settings import asbool, aslist
22
+ from pyramid_multiauth import MultiAuthenticationPolicy, MultiAuthPolicySelected
23
+
24
+ from kinto.core import cache, errors, metrics, permission, storage, utils
25
+ from kinto.core.events import ACTIONS, ResourceChanged, ResourceRead
26
+
27
+
28
+ try:
29
+ import newrelic.agent
30
+ except ImportError: # pragma: no cover
31
+ newrelic = None
32
+ try:
33
+ from werkzeug.middleware.profiler import ProfilerMiddleware
34
+ except ImportError: # pragma: no cover
35
+ ProfilerMiddleware = False
36
+ try:
37
+ import sentry_sdk
38
+ from sentry_sdk.integrations.logging import LoggingIntegration
39
+ from sentry_sdk.integrations.pyramid import PyramidIntegration
40
+ from sentry_sdk.integrations.sqlalchemy import SqlalchemyIntegration
41
+ except ImportError: # pragma: no cover
42
+ sentry_sdk = None
43
+
44
+
45
+ logger = logging.getLogger(__name__)
46
+ summary_logger = logging.getLogger("request.summary")
47
+
48
+
49
+ def setup_request_bound_data(config):
50
+ """Attach custom data on request object, and share it with parent
51
+ requests during batch."""
52
+
53
+ def attach_bound_data(request):
54
+ parent = getattr(request, "parent", None)
55
+ return parent.bound_data if parent else {}
56
+
57
+ config.add_request_method(attach_bound_data, name="bound_data", reify=True)
58
+
59
+
60
+ def setup_json_serializer(config):
61
+ import requests
62
+ import webob
63
+
64
+ # Monkey patch to use rapidjson
65
+ webob.request.json = utils.json
66
+ requests.models.json = utils.json
67
+
68
+ # Override json renderer using rapidjson
69
+ renderer = JSONRenderer(serializer=utils.json_serializer)
70
+ config.add_renderer("ultrajson", renderer) # See `kinto.core.Service`
71
+
72
+
73
+ def setup_csp_headers(config):
74
+ """Content-Security-Policy HTTP response header helps reduce XSS risks on
75
+ modern browsers by declaring, which dynamic resources are allowed to load.
76
+ On APIs, we disable everything.
77
+ """
78
+ disable_all = "default-src 'none'; frame-ancestors 'none'; base-uri 'none'; "
79
+
80
+ def on_new_response(event):
81
+ event.response.headers.setdefault("Content-Security-Policy", disable_all)
82
+
83
+ config.add_subscriber(on_new_response, NewResponse)
84
+
85
+
86
+ def restrict_http_methods_if_readonly(config):
87
+ """Prevent write operations if server is configured as read-only.
88
+
89
+ This is an additional layer of security on top of the verifications done
90
+ in :method:`kinto.core.Resource.register_service`, in case an installed
91
+ plugin does not take this setting into account in the definition of its
92
+ ad-hoc Pyramid views.
93
+ """
94
+ settings = config.get_settings()
95
+
96
+ def on_new_request(event):
97
+ if event.request.method.lower() not in ("get", "head", "options"):
98
+ raise HTTPMethodNotAllowed()
99
+
100
+ if asbool(settings["readonly"]):
101
+ config.add_subscriber(on_new_request, NewRequest)
102
+
103
+
104
+ def setup_version_redirection(config):
105
+ """Add a view which redirects to the current version of the API."""
106
+ settings = config.get_settings()
107
+ redirect_enabled = settings["version_prefix_redirect_enabled"]
108
+ version_prefix_redirection_enabled = asbool(redirect_enabled)
109
+ cache_seconds = int(settings["version_prefix_redirect_ttl_seconds"])
110
+
111
+ route_prefix = config.route_prefix
112
+ config.registry.route_prefix = route_prefix
113
+
114
+ # Redirect to the current version of the API if the prefix isn't used.
115
+ # Do not redirect if kinto.version_prefix_redirect_enabled is set to
116
+ # False.
117
+ if not version_prefix_redirection_enabled:
118
+ return
119
+
120
+ def _redirect_to_version_view(request):
121
+ if request.method.lower() == "options":
122
+ # CORS responses should always have status 200.
123
+ return utils.reapply_cors(request, Response())
124
+
125
+ querystring = request.url[(request.url.rindex(request.path) + len(request.path)) :]
126
+ redirect = f"/{route_prefix}{request.path}{querystring}"
127
+ resp = HTTPTemporaryRedirect(redirect)
128
+ if cache_seconds >= 0:
129
+ resp.cache_expires(cache_seconds)
130
+ raise resp
131
+
132
+ # Disable the route prefix passed by the app.
133
+ config.route_prefix = None
134
+
135
+ config.add_route(name="redirect_to_version", pattern=r"/{path:(?!v[0-9]+)[^\r\n]*}")
136
+
137
+ config.add_view(
138
+ view=_redirect_to_version_view,
139
+ route_name="redirect_to_version",
140
+ permission=NO_PERMISSION_REQUIRED,
141
+ )
142
+
143
+ config.route_prefix = route_prefix
144
+
145
+
146
+ def setup_authentication(config):
147
+ """Let pyramid_multiauth manage authentication and authorization
148
+ from configuration.
149
+ """
150
+ config.include("pyramid_multiauth")
151
+ settings = config.get_settings()
152
+
153
+ policies = aslist(settings["multiauth.policies"])
154
+ if "basicauth" in policies:
155
+ config.include("kinto.core.authentication")
156
+
157
+ # Track policy used, for prefixing user_id and for logging.
158
+ def on_policy_selected(event):
159
+ request = event.request
160
+ # If the authn policy has a name attribute, we use it rather
161
+ # than using the one specified in settings.
162
+ authn_type = getattr(event.policy, "name", event.policy_name.lower())
163
+ request.authn_type = authn_type
164
+ request.selected_userid = event.userid
165
+ # Add authentication info to context.
166
+ request.log_context(uid=event.userid, authn_type=authn_type)
167
+
168
+ config.add_subscriber(on_policy_selected, MultiAuthPolicySelected)
169
+
170
+
171
+ def setup_backoff(config):
172
+ """Attach HTTP requests/responses objects.
173
+
174
+ This is useful to attach objects to the request object for easier
175
+ access, and to pre-process responses.
176
+ """
177
+
178
+ def on_new_response(event):
179
+ # Add backoff in response headers.
180
+ backoff = config.registry.settings["backoff"]
181
+ if backoff is not None:
182
+ backoff_percentage = config.registry.settings["backoff_percentage"]
183
+ if backoff_percentage is not None:
184
+ if random.random() < (float(backoff_percentage) / 100.0):
185
+ event.response.headers["Backoff"] = str(backoff)
186
+ else:
187
+ event.response.headers["Backoff"] = str(backoff)
188
+
189
+ config.add_subscriber(on_new_response, NewResponse)
190
+
191
+
192
+ def setup_requests_scheme(config):
193
+ """Force server scheme, host and port at the application level."""
194
+ settings = config.get_settings()
195
+
196
+ http_scheme = settings["http_scheme"]
197
+ http_host = settings["http_host"]
198
+
199
+ def on_new_request(event):
200
+ if http_scheme:
201
+ event.request.scheme = http_scheme
202
+ if http_host:
203
+ event.request.host = http_host
204
+
205
+ if http_scheme or http_host:
206
+ config.add_subscriber(on_new_request, NewRequest)
207
+
208
+
209
+ def setup_deprecation(config):
210
+ config.add_tween("kinto.core.initialization._end_of_life_tween_factory")
211
+
212
+
213
+ def _end_of_life_tween_factory(handler, registry):
214
+ """Pyramid tween to handle service end of life."""
215
+ deprecation_msg = "The service you are trying to connect no longer exists at this location."
216
+
217
+ def eos_tween(request):
218
+ eos_date = registry.settings["eos"]
219
+ eos_url = registry.settings["eos_url"]
220
+ eos_message = registry.settings["eos_message"]
221
+ if not eos_date:
222
+ return handler(request)
223
+
224
+ eos_date = dateparser.parse(eos_date)
225
+ if eos_date > datetime.now():
226
+ code = "soft-eol"
227
+ request.response = handler(request)
228
+ else:
229
+ code = "hard-eol"
230
+ request.response = errors.http_error(
231
+ HTTPGone(),
232
+ errno=errors.ERRORS.SERVICE_DEPRECATED,
233
+ message=deprecation_msg,
234
+ )
235
+
236
+ errors.send_alert(request, eos_message, url=eos_url, code=code)
237
+ return utils.reapply_cors(request, request.response)
238
+
239
+ return eos_tween
240
+
241
+
242
+ def setup_storage(config):
243
+ settings = config.get_settings()
244
+
245
+ # Id generators by resource name.
246
+ config.registry.id_generators = {}
247
+ for key, value in settings.items():
248
+ m = re.match(r"^([^_]*)_?id_generator", key)
249
+ if m is None:
250
+ continue
251
+ resource_name = m.group(1)
252
+ id_generator = config.maybe_dotted(value)
253
+ config.registry.id_generators[resource_name] = id_generator()
254
+
255
+ storage_mod = settings["storage_backend"]
256
+ if not storage_mod:
257
+ return
258
+
259
+ storage_mod = config.maybe_dotted(storage_mod)
260
+ backend = storage_mod.load_from_config(config)
261
+ if not isinstance(backend, storage.StorageBase):
262
+ raise ConfigurationError(f"Invalid storage backend: {backend}")
263
+ config.registry.storage = backend
264
+
265
+ heartbeat = storage.heartbeat(backend)
266
+ config.registry.heartbeats["storage"] = heartbeat
267
+
268
+
269
+ def setup_permission(config):
270
+ settings = config.get_settings()
271
+ permission_mod = settings["permission_backend"]
272
+ if not permission_mod:
273
+ return
274
+
275
+ permission_mod = config.maybe_dotted(permission_mod)
276
+ backend = permission_mod.load_from_config(config)
277
+ if not isinstance(backend, permission.PermissionBase):
278
+ raise ConfigurationError(f"Invalid permission backend: {backend}")
279
+ config.registry.permission = backend
280
+
281
+ heartbeat = permission.heartbeat(backend)
282
+ config.registry.heartbeats["permission"] = heartbeat
283
+
284
+
285
+ def setup_cache(config):
286
+ settings = config.get_settings()
287
+ cache_mod = settings["cache_backend"]
288
+ if not cache_mod:
289
+ return
290
+
291
+ cache_mod = config.maybe_dotted(cache_mod)
292
+ backend = cache_mod.load_from_config(config)
293
+ if not isinstance(backend, cache.CacheBase):
294
+ raise ConfigurationError(f"Invalid cache backend: {backend}")
295
+ config.registry.cache = backend
296
+
297
+ heartbeat = cache.heartbeat(backend)
298
+ config.registry.heartbeats["cache"] = heartbeat
299
+
300
+
301
+ def setup_sentry(config):
302
+ settings = config.get_settings()
303
+
304
+ # Note: SENTRY_DSN and SENTRY_ENV env variables will override
305
+ # .ini values thanks to `load_default_settings()`.
306
+
307
+ dsn = settings["sentry_dsn"]
308
+ if dsn: # pragma: no cover
309
+ env_options = {}
310
+ env = settings["sentry_env"]
311
+ if env:
312
+ env_options["environment"] = env
313
+
314
+ breadcrumbs_level = settings["sentry_breadcrumbs_min_level"]
315
+ events_level = settings["sentry_events_min_level"]
316
+ sentry_sdk.init(
317
+ dsn,
318
+ integrations=[
319
+ PyramidIntegration(),
320
+ SqlalchemyIntegration(),
321
+ LoggingIntegration(
322
+ # Logs to be captured as breadcrumbs (debug and above by default)
323
+ level=breadcrumbs_level,
324
+ # Logs to be catpured as events (warning and above by default)
325
+ event_level=events_level,
326
+ ),
327
+ ],
328
+ **env_options,
329
+ )
330
+
331
+ def on_app_created(event):
332
+ msg = "Running {project_name} {project_version}.".format_map(settings)
333
+ sentry_sdk.capture_message(msg, "info")
334
+
335
+ config.add_subscriber(on_app_created, ApplicationCreated)
336
+
337
+
338
+ def install_middlewares(app, settings):
339
+ "Install a set of middlewares defined in the ini file on the given app."
340
+ # Setup new-relic.
341
+ if settings.get("newrelic_config"):
342
+ ini_file = settings["newrelic_config"]
343
+ env = settings["newrelic_env"]
344
+ newrelic.agent.initialize(ini_file, env)
345
+ app = newrelic.agent.WSGIApplicationWrapper(app)
346
+
347
+ # Adds the Werkzeug profiler.
348
+ if asbool(settings.get("profiler_enabled")):
349
+ profile_dir = settings["profiler_dir"]
350
+ app = ProfilerMiddleware(app, profile_dir=profile_dir, restrictions=("*kinto.core*"))
351
+
352
+ return app
353
+
354
+
355
+ def setup_logging(config):
356
+ """Setup structured logging, and emit `request.summary` event on each
357
+ request, as recommended by Mozilla Services standard:
358
+
359
+ * https://mana.mozilla.org/wiki/display/CLOUDSERVICES/Logging+Standard
360
+ * http://12factor.net/logs
361
+ """
362
+
363
+ def on_new_request(event):
364
+ request = event.request
365
+ # Save the time the request was received by the server.
366
+ event.request._received_at = utils.msec_time()
367
+
368
+ try:
369
+ # Pyramid fails if the URL contains invalid UTF-8 characters.
370
+ request_path = event.request.path
371
+ except UnicodeDecodeError:
372
+ raise errors.http_error(
373
+ HTTPBadRequest(),
374
+ errno=errors.ERRORS.INVALID_PARAMETERS,
375
+ message="Invalid URL path.",
376
+ )
377
+
378
+ rid = get_or_generate_request_id(headers=request.headers)
379
+ request_id_context.set(rid)
380
+
381
+ request.log_context(
382
+ agent=request.headers.get("User-Agent"),
383
+ path=request_path,
384
+ method=request.method,
385
+ lang=request.headers.get("Accept-Language"),
386
+ rid=rid,
387
+ errno=0,
388
+ )
389
+ qs = dict(errors.request_GET(request))
390
+ if qs:
391
+ request.log_context(querystring=qs)
392
+
393
+ if summary_logger.level == logging.DEBUG:
394
+ request.log_context(headers=dict(request.headers), body=request.body)
395
+
396
+ config.add_subscriber(on_new_request, NewRequest)
397
+
398
+ def on_new_response(event):
399
+ response = event.response
400
+ request = event.request
401
+
402
+ # Compute the request processing time in msec (-1 if unknown)
403
+ current = utils.msec_time()
404
+ duration = current - getattr(request, "_received_at", current - 1)
405
+ isotimestamp = datetime.fromtimestamp(current / 1000).isoformat()
406
+
407
+ # Bind infos for request summary logger.
408
+ request.log_context(time=isotimestamp, code=response.status_code, t=duration)
409
+
410
+ if summary_logger.level == logging.DEBUG:
411
+ request.log_context(response=dict(headers=dict(response.headers), body=response.body))
412
+
413
+ try:
414
+ # If error response, bind errno.
415
+ request.log_context(errno=response.errno)
416
+ except AttributeError:
417
+ pass
418
+
419
+ if not hasattr(request, "parent"):
420
+ # Ouput application request summary.
421
+ summary_logger.info("", extra=request.log_context())
422
+
423
+ config.add_subscriber(on_new_response, NewResponse)
424
+
425
+
426
+ def setup_metrics(config):
427
+ settings = config.get_settings()
428
+
429
+ # Register a no-op metrics service by default.
430
+ config.registry.registerUtility(metrics.NoOpMetricsService(), metrics.IMetricsService)
431
+
432
+ # This does not fully respect the Pyramid/ZCA patterns, but the rest of Kinto uses
433
+ # `registry.storage`, `registry.cache`, etc. Consistency seems more important.
434
+ config.registry.__class__.metrics = property(
435
+ lambda reg: reg.queryUtility(metrics.IMetricsService)
436
+ )
437
+
438
+ def deprecated_registry(self):
439
+ warnings.warn(
440
+ "``config.registry.statsd`` is now deprecated. Use ``config.registry.metrics`` instead.",
441
+ DeprecationWarning,
442
+ )
443
+ return self.metrics
444
+
445
+ config.registry.__class__.statsd = property(deprecated_registry)
446
+
447
+ def on_app_created(event):
448
+ config = event.app
449
+ metrics_service = config.registry.metrics
450
+
451
+ metrics.watch_execution_time(metrics_service, config.registry.cache, prefix="backend")
452
+ metrics.watch_execution_time(metrics_service, config.registry.storage, prefix="backend")
453
+ metrics.watch_execution_time(metrics_service, config.registry.permission, prefix="backend")
454
+
455
+ policy = config.registry.queryUtility(IAuthenticationPolicy)
456
+ if isinstance(policy, MultiAuthenticationPolicy):
457
+ for name, subpolicy in policy.get_policies():
458
+ metrics.watch_execution_time(
459
+ metrics_service, subpolicy, prefix="authentication", classname=name
460
+ )
461
+ else:
462
+ metrics.watch_execution_time(metrics_service, policy, prefix="authentication")
463
+
464
+ # Set cache metrics backend
465
+ cache_backend = config.registry.cache
466
+ if isinstance(cache_backend, cache.CacheBase):
467
+ cache_backend.set_metrics_backend(metrics_service)
468
+
469
+ config.add_subscriber(on_app_created, ApplicationCreated)
470
+
471
+ def on_new_response(event):
472
+ request = event.request
473
+ metrics_service = config.registry.metrics
474
+
475
+ # Count unique users.
476
+ user_id = request.prefixed_userid
477
+ if user_id:
478
+ auth, user_id = user_id.split(":", 1)
479
+ # Get rid of colons in metric packet (see #1282 and #3571).
480
+ user_id = user_id.replace(":", ".")
481
+ metrics_service.count("users", unique=[("auth", auth), ("userid", user_id)])
482
+
483
+ status = event.response.status_code
484
+
485
+ if status >= 400:
486
+ # Prevent random values of 404 responses to become label values.
487
+ request_matchdict = {}
488
+ else:
489
+ request_matchdict = dict(request.matchdict or {})
490
+
491
+ # Add extra labels to metrics, based on fields extracted from the request matchdict.
492
+ metrics_matchdict_fields = aslist(settings["metrics_matchdict_fields"])
493
+ # Turn the `id` field of object endpoints into `{resource}_id` (eg. `mushroom_id`, `bucket_id`)
494
+ enhanced_matchdict = request_matchdict
495
+ try:
496
+ enhanced_matchdict[request.current_resource_name + "_id"] = enhanced_matchdict.get(
497
+ "id", ""
498
+ )
499
+ except AttributeError:
500
+ # Not on a resource.
501
+ pass
502
+ metrics_matchdict_labels = [
503
+ (field, enhanced_matchdict.get(field, "")) for field in metrics_matchdict_fields
504
+ ]
505
+
506
+ service = request.current_service
507
+ if service:
508
+ # Use the service name as endpoint if available.
509
+ endpoint = service.name
510
+ elif route := request.matched_route:
511
+ # Use the route name as endpoint if we're not on a Cornice service.
512
+ endpoint = route.name
513
+ else:
514
+ endpoint = (
515
+ "unnamed" if status != 404 else "unknown"
516
+ ) # Do not multiply cardinality for unknown endpoints.
517
+
518
+ request_labels = [
519
+ ("method", request.method.lower()),
520
+ ("endpoint", endpoint),
521
+ ] + metrics_matchdict_labels
522
+
523
+ # Count served requests.
524
+ metrics_service.count("request_summary", unique=request_labels + [("status", str(status))])
525
+
526
+ try:
527
+ current = utils.msec_time()
528
+ duration = (current - request._received_at) / 1000
529
+ metrics_service.timer(
530
+ "request_duration_seconds",
531
+ value=duration,
532
+ labels=request_labels,
533
+ )
534
+ except AttributeError: # pragma: no cover
535
+ # Logging was not setup in this Kinto app (unlikely but possible)
536
+ pass
537
+
538
+ # Observe response size.
539
+ metrics_service.observe(
540
+ "request_size", len(event.response.body or b""), labels=request_labels
541
+ )
542
+
543
+ # Count authentication verifications.
544
+ try:
545
+ metrics_service.count("authentication", unique=[("type", request.authn_type)])
546
+ except AttributeError:
547
+ # Not authenticated
548
+ pass
549
+
550
+ config.add_subscriber(on_new_response, NewResponse)
551
+
552
+
553
+ class EventActionFilter:
554
+ def __init__(self, actions, config):
555
+ actions = ACTIONS.from_string_list(actions)
556
+ self.actions = [action.value for action in actions]
557
+
558
+ def phash(self):
559
+ return f"for_actions = {','.join(self.actions)}"
560
+
561
+ def __call__(self, event):
562
+ action = event.payload.get("action")
563
+ return not action or action in self.actions
564
+
565
+
566
+ class EventResourceFilter:
567
+ def __init__(self, resources, config):
568
+ self.resources = resources
569
+
570
+ def phash(self):
571
+ return f"for_resources = {','.join(self.resources)}"
572
+
573
+ def __call__(self, event):
574
+ resource = event.payload.get("resource_name")
575
+ return not resource or not self.resources or resource in self.resources
576
+
577
+
578
+ def setup_listeners(config):
579
+ # Register basic subscriber predicates, to filter events.
580
+ config.add_subscriber_predicate("for_actions", EventActionFilter)
581
+ config.add_subscriber_predicate("for_resources", EventResourceFilter)
582
+
583
+ write_actions = (ACTIONS.CREATE, ACTIONS.UPDATE, ACTIONS.DELETE)
584
+ settings = config.get_settings()
585
+ settings_prefix = settings.get("settings_prefix", "")
586
+ listeners = aslist(settings["event_listeners"])
587
+
588
+ for name in listeners:
589
+ logger.info(f"Setting up '{name}' listener")
590
+ prefix = f"event_listeners.{name}."
591
+
592
+ try:
593
+ listener_mod = config.maybe_dotted(name)
594
+ prefix = f"event_listeners.{name.split('.')[-1]}."
595
+ listener = listener_mod.load_from_config(config, prefix)
596
+ except (ImportError, AttributeError):
597
+ module_setting = prefix + "use"
598
+ # Read from ENV or settings.
599
+ module_value = utils.read_env(
600
+ f"{settings_prefix}.{module_setting}", settings.get(module_setting)
601
+ )
602
+ listener_mod = config.maybe_dotted(module_value)
603
+ listener = listener_mod.load_from_config(config, prefix)
604
+
605
+ wrapped_listener = metrics.listener_with_timer(
606
+ config, f"listeners.{name}", listener.__call__
607
+ )
608
+
609
+ # Optional filter by event action.
610
+ actions_setting = prefix + "actions"
611
+ # Read from ENV or settings.
612
+ actions_value = utils.read_env(
613
+ f"{settings_prefix}.{actions_setting}", settings.get(actions_setting, "")
614
+ )
615
+ actions = aslist(actions_value)
616
+ if len(actions) > 0:
617
+ actions = ACTIONS.from_string_list(actions)
618
+ else:
619
+ actions = write_actions
620
+
621
+ # Optional filter by event resource name.
622
+ resource_setting = prefix + "resources"
623
+ # Read from ENV or settings.
624
+ resource_value = utils.read_env(
625
+ f"{settings_prefix}.{resource_setting}", settings.get(resource_setting, "")
626
+ )
627
+ resource_names = aslist(resource_value)
628
+
629
+ # Pyramid event predicates.
630
+ options = dict(for_actions=actions, for_resources=resource_names)
631
+
632
+ if ACTIONS.READ in actions:
633
+ config.add_subscriber(wrapped_listener, ResourceRead, **options)
634
+ actions = [a for a in actions if a != ACTIONS.READ]
635
+
636
+ if len(actions) > 0:
637
+ config.add_subscriber(wrapped_listener, ResourceChanged, **options)
638
+
639
+
640
+ def load_default_settings(config, default_settings):
641
+ """Read settings provided in Paste ini file, set default values and
642
+ replace if defined as environment variable.
643
+ """
644
+ settings = config.get_settings()
645
+
646
+ settings_prefix = settings["settings_prefix"]
647
+
648
+ def _prefixed_keys(key):
649
+ unprefixed = key
650
+ if key.startswith(settings_prefix + "."):
651
+ unprefixed = key.split(".", 1)[1]
652
+ project_prefix = f"{settings_prefix}.{unprefixed}"
653
+ return unprefixed, project_prefix
654
+
655
+ # Fill settings with default values if not defined.
656
+ for key, default_value in sorted(default_settings.items()):
657
+ unprefixed, project_prefix = keys = _prefixed_keys(key)
658
+ is_defined = len(set(settings.keys()).intersection(set(keys))) > 0
659
+ if not is_defined:
660
+ settings[unprefixed] = default_value
661
+
662
+ for key, value in sorted(settings.items()):
663
+ value = utils.native_value(value)
664
+ unprefixed, project_prefix = keys = _prefixed_keys(key)
665
+
666
+ # Fail if not only one is defined.
667
+ defined = set(settings.keys()).intersection(set(keys))
668
+ distinct_values = set([str(settings[d]) for d in defined])
669
+
670
+ if len(defined) > 1 and len(distinct_values) > 1:
671
+ names = "', '".join(defined)
672
+ raise ValueError(f"Settings '{names}' are in conflict.")
673
+
674
+ # Override settings from OS env values.
675
+ # e.g. HTTP_PORT, READINGLIST_HTTP_PORT, KINTO_HTTP_PORT
676
+ from_env = utils.read_env(unprefixed, value)
677
+ from_env = utils.read_env(project_prefix, from_env)
678
+
679
+ settings[unprefixed] = from_env
680
+
681
+ config.add_settings(settings)
682
+
683
+
684
+ def initialize(config, version=None, settings_prefix="", default_settings=None):
685
+ """Initialize kinto.core with the given configuration, version and project
686
+ name.
687
+
688
+ This will basically include kinto.core in Pyramid and set route prefix
689
+ based on the specified version.
690
+
691
+ :param config: Pyramid configuration
692
+ :type config: :class:`~pyramid:pyramid.config.Configurator`
693
+ :param str version: Current project version (e.g. '0.0.1') if not defined
694
+ in application settings.
695
+ :param str settings_prefix: Project name if not defined
696
+ in application settings.
697
+ :param dict default_settings: Override kinto.core default settings values.
698
+ """
699
+ from kinto.core import DEFAULT_SETTINGS
700
+
701
+ settings = config.get_settings()
702
+
703
+ settings_prefix = (
704
+ settings.pop("kinto.settings_prefix", settings.get("settings_prefix")) or settings_prefix
705
+ )
706
+ settings["settings_prefix"] = settings_prefix
707
+ if not settings_prefix:
708
+ warnings.warn("No value specified for `settings_prefix`")
709
+
710
+ kinto_core_defaults = {**DEFAULT_SETTINGS}
711
+
712
+ if default_settings:
713
+ kinto_core_defaults.update(default_settings)
714
+
715
+ load_default_settings(config, kinto_core_defaults)
716
+
717
+ http_scheme = settings["http_scheme"]
718
+ if http_scheme != "https":
719
+ warnings.warn("HTTPS is not enabled")
720
+
721
+ # Override project version from settings.
722
+ project_version = settings.get("project_version") or version
723
+ if not project_version:
724
+ error_msg = f"Invalid project version: {project_version}"
725
+ raise ConfigurationError(error_msg)
726
+ settings["project_version"] = project_version = str(project_version)
727
+
728
+ # HTTP API version.
729
+ http_api_version = settings.get("http_api_version")
730
+ if http_api_version is None:
731
+ # The API version is derivated from the module version if not provided.
732
+ http_api_version = ".".join(project_version.split(".")[0:2])
733
+ settings["http_api_version"] = http_api_version = str(http_api_version)
734
+ api_version = f"v{http_api_version.split('.')[0]}"
735
+
736
+ # Include kinto.core views with the correct api version prefix.
737
+ config.include("kinto.core", route_prefix=api_version)
738
+ config.route_prefix = api_version