c2cwsgiutils 6.1.0.dev105__py3-none-any.whl → 6.1.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- c2cwsgiutils/__init__.py +14 -11
- c2cwsgiutils/acceptance/__init__.py +2 -3
- c2cwsgiutils/acceptance/connection.py +1 -2
- c2cwsgiutils/acceptance/image.py +17 -11
- c2cwsgiutils/acceptance/package-lock.json +306 -213
- c2cwsgiutils/acceptance/package.json +2 -2
- c2cwsgiutils/acceptance/print.py +7 -3
- c2cwsgiutils/acceptance/utils.py +1 -3
- c2cwsgiutils/auth.py +27 -25
- c2cwsgiutils/broadcast/__init__.py +15 -16
- c2cwsgiutils/broadcast/interface.py +3 -3
- c2cwsgiutils/broadcast/local.py +1 -0
- c2cwsgiutils/broadcast/redis.py +13 -12
- c2cwsgiutils/client_info.py +19 -1
- c2cwsgiutils/coverage_setup.py +4 -3
- c2cwsgiutils/db.py +35 -41
- c2cwsgiutils/db_maintenance_view.py +13 -13
- c2cwsgiutils/debug/__init__.py +2 -2
- c2cwsgiutils/debug/_listeners.py +2 -7
- c2cwsgiutils/debug/_views.py +20 -12
- c2cwsgiutils/debug/utils.py +9 -9
- c2cwsgiutils/errors.py +13 -15
- c2cwsgiutils/health_check.py +24 -30
- c2cwsgiutils/index.py +34 -13
- c2cwsgiutils/loader.py +21 -2
- c2cwsgiutils/logging_view.py +12 -12
- c2cwsgiutils/models_graph.py +0 -1
- c2cwsgiutils/pretty_json.py +0 -1
- c2cwsgiutils/prometheus.py +10 -10
- c2cwsgiutils/pyramid.py +0 -1
- c2cwsgiutils/pyramid_logging.py +1 -1
- c2cwsgiutils/redis_stats.py +9 -9
- c2cwsgiutils/redis_utils.py +19 -18
- c2cwsgiutils/request_tracking/__init__.py +13 -13
- c2cwsgiutils/request_tracking/_sql.py +0 -1
- c2cwsgiutils/scripts/genversion.py +5 -5
- c2cwsgiutils/scripts/stats_db.py +19 -17
- c2cwsgiutils/scripts/test_print.py +5 -5
- c2cwsgiutils/sentry.py +55 -20
- c2cwsgiutils/services.py +2 -2
- c2cwsgiutils/setup_process.py +0 -1
- c2cwsgiutils/sql_profiler/__init__.py +5 -6
- c2cwsgiutils/sql_profiler/_impl.py +18 -17
- c2cwsgiutils/sqlalchemylogger/README.md +30 -13
- c2cwsgiutils/sqlalchemylogger/handlers.py +12 -11
- c2cwsgiutils/stats_pyramid/__init__.py +1 -5
- c2cwsgiutils/stats_pyramid/_db_spy.py +2 -2
- c2cwsgiutils/stats_pyramid/_pyramid_spy.py +12 -1
- c2cwsgiutils/templates/index.html.mako +4 -1
- c2cwsgiutils/version.py +11 -5
- {c2cwsgiutils-6.1.0.dev105.dist-info → c2cwsgiutils-6.1.7.dist-info}/LICENSE +1 -1
- {c2cwsgiutils-6.1.0.dev105.dist-info → c2cwsgiutils-6.1.7.dist-info}/METADATA +18 -6
- c2cwsgiutils-6.1.7.dist-info/RECORD +67 -0
- {c2cwsgiutils-6.1.0.dev105.dist-info → c2cwsgiutils-6.1.7.dist-info}/WHEEL +1 -1
- c2cwsgiutils-6.1.0.dev105.dist-info/RECORD +0 -67
- {c2cwsgiutils-6.1.0.dev105.dist-info → c2cwsgiutils-6.1.7.dist-info}/entry_points.txt +0 -0
c2cwsgiutils/db.py
CHANGED
@@ -5,7 +5,7 @@ import re
|
|
5
5
|
import warnings
|
6
6
|
from collections.abc import Iterable
|
7
7
|
from re import Pattern
|
8
|
-
from typing import
|
8
|
+
from typing import Any, Callable, Optional, Union, cast
|
9
9
|
|
10
10
|
import pyramid.config
|
11
11
|
import pyramid.config.settings
|
@@ -17,18 +17,14 @@ import zope.sqlalchemy
|
|
17
17
|
from sqlalchemy import engine_from_config
|
18
18
|
from zope.sqlalchemy import register
|
19
19
|
|
20
|
-
|
21
|
-
|
22
|
-
sessionmaker = sqlalchemy.orm.sessionmaker[sqlalchemy.orm.Session]
|
23
|
-
else:
|
24
|
-
scoped_session = sqlalchemy.orm.scoped_session
|
25
|
-
sessionmaker = sqlalchemy.orm.sessionmaker
|
20
|
+
_scoped_session = sqlalchemy.orm.scoped_session[sqlalchemy.orm.Session]
|
21
|
+
_sessionmaker = sqlalchemy.orm.sessionmaker[sqlalchemy.orm.Session] # pylint: disable=unsubscriptable-object
|
26
22
|
|
27
23
|
|
28
|
-
|
29
|
-
|
24
|
+
_LOG = logging.getLogger(__name__)
|
25
|
+
_RE_COMPILE: Callable[[str], Pattern[str]] = re.compile
|
30
26
|
|
31
|
-
|
27
|
+
FORCE_READONLY = False
|
32
28
|
|
33
29
|
|
34
30
|
class Tweens:
|
@@ -45,7 +41,7 @@ def setup_session(
|
|
45
41
|
force_master: Optional[Iterable[str]] = None,
|
46
42
|
force_slave: Optional[Iterable[str]] = None,
|
47
43
|
) -> tuple[
|
48
|
-
Union[sqlalchemy.orm.Session,
|
44
|
+
Union[sqlalchemy.orm.Session, _scoped_session],
|
49
45
|
sqlalchemy.engine.Engine,
|
50
46
|
sqlalchemy.engine.Engine,
|
51
47
|
]:
|
@@ -60,8 +56,7 @@ def setup_session(
|
|
60
56
|
Those parameters are lists of regex that are going to be matched against "{VERB} {PATH}". Warning, the
|
61
57
|
path includes the route_prefix.
|
62
58
|
|
63
|
-
|
64
|
-
|
59
|
+
Arguments:
|
65
60
|
config: The pyramid Configuration object
|
66
61
|
master_prefix: The prefix for the master connection configuration entries in the application \
|
67
62
|
settings
|
@@ -84,7 +79,7 @@ def setup_session(
|
|
84
79
|
|
85
80
|
# Setup a slave DB connection and add a tween to use it.
|
86
81
|
if settings[master_prefix + ".url"] != settings.get(slave_prefix + ".url"):
|
87
|
-
|
82
|
+
_LOG.info("Using a slave DB for reading %s", master_prefix)
|
88
83
|
ro_engine = sqlalchemy.engine_from_config(config.get_settings(), slave_prefix + ".")
|
89
84
|
ro_engine.c2c_name = slave_prefix # type: ignore
|
90
85
|
tween_name = master_prefix.replace(".", "_")
|
@@ -105,7 +100,7 @@ def create_session(
|
|
105
100
|
force_master: Optional[Iterable[str]] = None,
|
106
101
|
force_slave: Optional[Iterable[str]] = None,
|
107
102
|
**engine_config: Any,
|
108
|
-
) -> Union[sqlalchemy.orm.Session,
|
103
|
+
) -> Union[sqlalchemy.orm.Session, _scoped_session]:
|
109
104
|
"""
|
110
105
|
Create a SQLAlchemy session.
|
111
106
|
|
@@ -117,8 +112,7 @@ def create_session(
|
|
117
112
|
Those parameters are lists of regex that are going to be matched against "{VERB} {PATH}". Warning, the
|
118
113
|
path includes the route_prefix.
|
119
114
|
|
120
|
-
|
121
|
-
|
115
|
+
Arguments:
|
122
116
|
config: The pyramid Configuration object. If None, only master is used
|
123
117
|
name: The name of the check
|
124
118
|
url: The URL for the master DB
|
@@ -140,7 +134,7 @@ def create_session(
|
|
140
134
|
|
141
135
|
# Setup a slave DB connection and add a tween to use it.
|
142
136
|
if url != slave_url and config is not None:
|
143
|
-
|
137
|
+
_LOG.info("Using a slave DB for reading %s", name)
|
144
138
|
ro_engine = sqlalchemy.create_engine(slave_url, **engine_config)
|
145
139
|
_add_tween(config, name, db_session, force_master, force_slave)
|
146
140
|
rw_engine.c2c_name = name + "_master" # type: ignore
|
@@ -157,15 +151,15 @@ def create_session(
|
|
157
151
|
def _add_tween(
|
158
152
|
config: pyramid.config.Configurator,
|
159
153
|
name: str,
|
160
|
-
db_session:
|
154
|
+
db_session: _scoped_session,
|
161
155
|
force_master: Optional[Iterable[str]],
|
162
156
|
force_slave: Optional[Iterable[str]],
|
163
157
|
) -> None:
|
164
158
|
master_paths: Iterable[Pattern[str]] = (
|
165
|
-
list(map(
|
159
|
+
list(map(_RE_COMPILE, force_master)) if force_master is not None else []
|
166
160
|
)
|
167
161
|
slave_paths: Iterable[Pattern[str]] = (
|
168
|
-
list(map(
|
162
|
+
list(map(_RE_COMPILE, force_slave)) if force_slave is not None else []
|
169
163
|
)
|
170
164
|
|
171
165
|
def db_chooser_tween_factory(
|
@@ -182,18 +176,18 @@ def _add_tween(
|
|
182
176
|
old = session.bind
|
183
177
|
method_path: Any = f"{request.method} {request.path}"
|
184
178
|
has_force_master = any(r.match(method_path) for r in master_paths)
|
185
|
-
if
|
179
|
+
if FORCE_READONLY or (
|
186
180
|
not has_force_master
|
187
181
|
and (request.method in ("GET", "OPTIONS") or any(r.match(method_path) for r in slave_paths))
|
188
182
|
):
|
189
|
-
|
183
|
+
_LOG.debug(
|
190
184
|
"Using %s database for: %s",
|
191
185
|
db_session.c2c_ro_bind.c2c_name, # type: ignore
|
192
186
|
method_path,
|
193
187
|
)
|
194
188
|
session.bind = db_session.c2c_ro_bind # type: ignore
|
195
189
|
else:
|
196
|
-
|
190
|
+
_LOG.debug(
|
197
191
|
"Using %s database for: %s",
|
198
192
|
db_session.c2c_rw_bind.c2c_name, # type: ignore
|
199
193
|
method_path,
|
@@ -211,7 +205,7 @@ def _add_tween(
|
|
211
205
|
config.add_tween("c2cwsgiutils.db.tweens." + name, over="pyramid_tm.tm_tween_factory")
|
212
206
|
|
213
207
|
|
214
|
-
class SessionFactory(
|
208
|
+
class SessionFactory(_sessionmaker):
|
215
209
|
"""The custom session factory that manage the read only and read write sessions."""
|
216
210
|
|
217
211
|
def __init__(
|
@@ -223,42 +217,43 @@ class SessionFactory(sessionmaker):
|
|
223
217
|
):
|
224
218
|
super().__init__()
|
225
219
|
self.master_paths: Iterable[Pattern[str]] = (
|
226
|
-
list(map(
|
220
|
+
list(map(_RE_COMPILE, force_master)) if force_master else []
|
227
221
|
)
|
228
|
-
self.slave_paths: Iterable[Pattern[str]] = list(map(
|
222
|
+
self.slave_paths: Iterable[Pattern[str]] = list(map(_RE_COMPILE, force_slave)) if force_slave else []
|
229
223
|
self.ro_engine = ro_engine
|
230
224
|
self.rw_engine = rw_engine
|
231
225
|
|
232
226
|
def engine_name(self, readwrite: bool) -> str:
|
227
|
+
"""Get the engine name."""
|
233
228
|
if readwrite:
|
234
229
|
return cast(str, self.rw_engine.c2c_name) # type: ignore
|
235
230
|
return cast(str, self.ro_engine.c2c_name) # type: ignore
|
236
231
|
|
237
232
|
def __call__( # type: ignore
|
238
233
|
self, request: Optional[pyramid.request.Request], readwrite: Optional[bool] = None, **local_kw: Any
|
239
|
-
) ->
|
234
|
+
) -> _scoped_session:
|
240
235
|
if readwrite is not None:
|
241
|
-
if readwrite and not
|
242
|
-
|
236
|
+
if readwrite and not FORCE_READONLY:
|
237
|
+
_LOG.debug("Using %s database", self.rw_engine.c2c_name) # type: ignore
|
243
238
|
self.configure(bind=self.rw_engine)
|
244
239
|
else:
|
245
|
-
|
240
|
+
_LOG.debug("Using %s database", self.ro_engine.c2c_name) # type: ignore
|
246
241
|
self.configure(bind=self.ro_engine)
|
247
242
|
else:
|
248
243
|
assert request is not None
|
249
244
|
method_path: str = f"{request.method} {request.path}" if request is not None else ""
|
250
245
|
has_force_master = any(r.match(method_path) for r in self.master_paths)
|
251
|
-
if
|
246
|
+
if FORCE_READONLY or (
|
252
247
|
not has_force_master
|
253
248
|
and (
|
254
249
|
request.method in ("GET", "OPTIONS")
|
255
250
|
or any(r.match(method_path) for r in self.slave_paths)
|
256
251
|
)
|
257
252
|
):
|
258
|
-
|
253
|
+
_LOG.debug("Using %s database for: %s", self.ro_engine.c2c_name, method_path) # type: ignore
|
259
254
|
self.configure(bind=self.ro_engine)
|
260
255
|
else:
|
261
|
-
|
256
|
+
_LOG.debug("Using %s database for: %s", self.rw_engine.c2c_name, method_path) # type: ignore
|
262
257
|
self.configure(bind=self.rw_engine)
|
263
258
|
return super().__call__(**local_kw) # type: ignore
|
264
259
|
|
@@ -272,15 +267,15 @@ def get_engine(
|
|
272
267
|
|
273
268
|
def get_session_factory(
|
274
269
|
engine: sqlalchemy.engine.Engine,
|
275
|
-
) ->
|
270
|
+
) -> _sessionmaker:
|
276
271
|
"""Get the session factory from the engine."""
|
277
|
-
factory =
|
272
|
+
factory = _sessionmaker()
|
278
273
|
factory.configure(bind=engine)
|
279
274
|
return factory
|
280
275
|
|
281
276
|
|
282
277
|
def get_tm_session(
|
283
|
-
session_factory:
|
278
|
+
session_factory: _sessionmaker,
|
284
279
|
transaction_manager: transaction.TransactionManager,
|
285
280
|
) -> sqlalchemy.orm.Session:
|
286
281
|
"""
|
@@ -348,7 +343,7 @@ def get_tm_session_pyramid(
|
|
348
343
|
session_factory: SessionFactory,
|
349
344
|
transaction_manager: transaction.TransactionManager,
|
350
345
|
request: pyramid.request.Request,
|
351
|
-
) ->
|
346
|
+
) -> _scoped_session:
|
352
347
|
"""
|
353
348
|
Get a ``sqlalchemy.orm.Session`` instance backed by a transaction.
|
354
349
|
|
@@ -369,8 +364,7 @@ def init(
|
|
369
364
|
"""
|
370
365
|
Initialize the database for a Pyramid app.
|
371
366
|
|
372
|
-
|
373
|
-
|
367
|
+
Arguments:
|
374
368
|
config: The pyramid Configuration object
|
375
369
|
master_prefix: The prefix for the master connection configuration entries in the application \
|
376
370
|
settings
|
@@ -392,7 +386,7 @@ def init(
|
|
392
386
|
|
393
387
|
# Setup a slave DB connection and add a tween to use it.
|
394
388
|
if slave_prefix and settings[master_prefix + ".url"] != settings.get(slave_prefix + ".url"):
|
395
|
-
|
389
|
+
_LOG.info("Using a slave DB for reading %s", master_prefix)
|
396
390
|
ro_engine = get_engine(config.get_settings(), slave_prefix + ".")
|
397
391
|
ro_engine.c2c_name = slave_prefix # type: ignore
|
398
392
|
else:
|
@@ -7,10 +7,10 @@ import pyramid.request
|
|
7
7
|
|
8
8
|
from c2cwsgiutils import auth, broadcast, config_utils, db, redis_utils
|
9
9
|
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
10
|
+
_LOG = logging.getLogger(__name__)
|
11
|
+
_CONFIG_KEY = "c2c.db_maintenance_view_enabled"
|
12
|
+
_ENV_KEY = "C2C_DB_MAINTENANCE_VIEW_ENABLED"
|
13
|
+
_REDIS_PREFIX = "c2c_db_maintenance_"
|
14
14
|
|
15
15
|
|
16
16
|
def install_subscriber(config: pyramid.config.Configurator) -> None:
|
@@ -21,7 +21,7 @@ def install_subscriber(config: pyramid.config.Configurator) -> None:
|
|
21
21
|
|
22
22
|
def includeme(config: pyramid.config.Configurator) -> None:
|
23
23
|
"""Install the view to configure the loggers, if configured to do so."""
|
24
|
-
if auth.is_enabled(config,
|
24
|
+
if auth.is_enabled(config, _ENV_KEY, _CONFIG_KEY):
|
25
25
|
config.add_route(
|
26
26
|
"c2c_db_maintenance",
|
27
27
|
config_utils.get_base_path(config) + r"/db/maintenance",
|
@@ -29,7 +29,7 @@ def includeme(config: pyramid.config.Configurator) -> None:
|
|
29
29
|
)
|
30
30
|
config.add_view(_db_maintenance, route_name="c2c_db_maintenance", renderer="fast_json", http_cache=0)
|
31
31
|
_restore(config)
|
32
|
-
|
32
|
+
_LOG.info("Enabled the /db/maintenance API")
|
33
33
|
|
34
34
|
|
35
35
|
def _db_maintenance(request: pyramid.request.Request) -> Mapping[str, Any]:
|
@@ -38,7 +38,7 @@ def _db_maintenance(request: pyramid.request.Request) -> Mapping[str, Any]:
|
|
38
38
|
if readonly_param is not None:
|
39
39
|
readonly = readonly_param.lower() == "true"
|
40
40
|
|
41
|
-
|
41
|
+
_LOG.critical("Readonly DB status changed from %s to %s", db.FORCE_READONLY, readonly)
|
42
42
|
_set_readonly(value=readonly)
|
43
43
|
_store(request.registry.settings, readonly)
|
44
44
|
return {"status": 200, "readonly": readonly}
|
@@ -51,7 +51,7 @@ def _db_maintenance(request: pyramid.request.Request) -> Mapping[str, Any]:
|
|
51
51
|
|
52
52
|
@broadcast.decorator(expect_answers=True)
|
53
53
|
def _set_readonly(value: bool) -> bool:
|
54
|
-
db.
|
54
|
+
db.FORCE_READONLY = value
|
55
55
|
return True
|
56
56
|
|
57
57
|
|
@@ -59,24 +59,24 @@ def _restore(config: pyramid.config.Configurator) -> None:
|
|
59
59
|
try:
|
60
60
|
readonly = _get_redis_value(config.get_settings())
|
61
61
|
if readonly is not None:
|
62
|
-
|
63
|
-
db.
|
62
|
+
_LOG.debug("Restoring readonly DB status to %s", readonly)
|
63
|
+
db.FORCE_READONLY = readonly == "true"
|
64
64
|
except ImportError:
|
65
65
|
pass # don't have redis
|
66
66
|
except Exception: # pylint: disable=broad-except
|
67
67
|
# survive an error since crashing now can have bad consequences for the service. :/
|
68
|
-
|
68
|
+
_LOG.error("Cannot restore readonly DB status.", exc_info=True)
|
69
69
|
|
70
70
|
|
71
71
|
def _store(settings: Mapping[str, Any], readonly: bool) -> None:
|
72
72
|
master, _, _ = redis_utils.get(settings)
|
73
73
|
if master is not None:
|
74
|
-
master.set(
|
74
|
+
master.set(_REDIS_PREFIX + "force_readonly", "true" if readonly else "false")
|
75
75
|
|
76
76
|
|
77
77
|
def _get_redis_value(settings: Mapping[str, Any]) -> Optional[str]:
|
78
78
|
_, slave, _ = redis_utils.get(settings)
|
79
79
|
if slave is not None:
|
80
|
-
value = slave.get(
|
80
|
+
value = slave.get(_REDIS_PREFIX + "force_readonly")
|
81
81
|
return str(value) if value else None
|
82
82
|
return None
|
c2cwsgiutils/debug/__init__.py
CHANGED
@@ -23,7 +23,7 @@ def init(config: pyramid.config.Configurator) -> None:
|
|
23
23
|
def includeme(config: pyramid.config.Configurator) -> None:
|
24
24
|
"""Initialize the debug tools."""
|
25
25
|
if auth.is_enabled(config, ENV_KEY, CONFIG_KEY):
|
26
|
-
from c2cwsgiutils.debug import _views
|
26
|
+
from c2cwsgiutils.debug import _views # pylint: disable=import-outside-toplevel
|
27
27
|
|
28
28
|
init_daemon(config)
|
29
29
|
_views.init(config)
|
@@ -37,6 +37,6 @@ def init_daemon(config: Optional[pyramid.config.Configurator] = None) -> None:
|
|
37
37
|
those requests.
|
38
38
|
"""
|
39
39
|
if config_utils.env_or_config(config, ENV_KEY, CONFIG_KEY, type_=config_utils.config_bool):
|
40
|
-
from c2cwsgiutils.debug import _listeners
|
40
|
+
from c2cwsgiutils.debug import _listeners # pylint: disable=import-outside-toplevel
|
41
41
|
|
42
42
|
_listeners.init()
|
c2cwsgiutils/debug/_listeners.py
CHANGED
@@ -68,12 +68,7 @@ def _dump_memory_impl(
|
|
68
68
|
and not (FILES_FIELDS - set(obj["globals"].keys()))
|
69
69
|
):
|
70
70
|
python_internal = True
|
71
|
-
if
|
72
|
-
python_internal
|
73
|
-
and not python_internals_map
|
74
|
-
or not python_internal
|
75
|
-
and python_internals_map
|
76
|
-
):
|
71
|
+
if python_internal != python_internals_map:
|
77
72
|
continue
|
78
73
|
size = get_size(obj) / 1024
|
79
74
|
if len(biggest_objects) < limit or size > biggest_objects[0][0]:
|
@@ -90,7 +85,7 @@ def _dump_memory_impl(
|
|
90
85
|
for i in sorted(mod_counts.items(), key=lambda x: -x[1])[:limit]
|
91
86
|
]
|
92
87
|
elif analyze_type == "linecache":
|
93
|
-
import linecache
|
88
|
+
import linecache # pylint: disable=import-outside-toplevel
|
94
89
|
|
95
90
|
cache = linecache.cache
|
96
91
|
result[analyze_type]["biggest_objects"] = sorted(
|
c2cwsgiutils/debug/_views.py
CHANGED
@@ -16,8 +16,8 @@ from pyramid.httpexceptions import HTTPException, exception_response
|
|
16
16
|
from c2cwsgiutils import auth, broadcast, config_utils
|
17
17
|
from c2cwsgiutils.debug.utils import dump_memory_maps, get_size
|
18
18
|
|
19
|
-
|
20
|
-
|
19
|
+
_LOG = logging.getLogger(__name__)
|
20
|
+
_SPACE_RE = re.compile(r" +")
|
21
21
|
|
22
22
|
|
23
23
|
def _beautify_stacks(source: list[Mapping[str, Any]]) -> list[Mapping[str, Any]]:
|
@@ -47,7 +47,7 @@ def _dump_memory(request: pyramid.request.Request) -> list[Mapping[str, Any]]:
|
|
47
47
|
auth.auth_view(request)
|
48
48
|
limit = int(request.params.get("limit", "30"))
|
49
49
|
analyze_type = request.params.get("analyze_type")
|
50
|
-
python_internals_map = request.params.get("python_internals_map", "0").lower() in ("
|
50
|
+
python_internals_map = request.params.get("python_internals_map", "0").lower() in ("1", "true", "on")
|
51
51
|
result = broadcast.broadcast(
|
52
52
|
"c2c_dump_memory",
|
53
53
|
params={"limit": limit, "analyze_type": analyze_type, "python_internals_map": python_internals_map},
|
@@ -75,12 +75,12 @@ def _dump_memory_diff(request: pyramid.request.Request) -> list[Any]:
|
|
75
75
|
|
76
76
|
# warm-up run
|
77
77
|
try:
|
78
|
-
if "no_warmup"
|
78
|
+
if request.params.get("no_warmup", "0").lower() in ("1", "true", "on"):
|
79
79
|
request.invoke_subrequest(sub_request)
|
80
80
|
except Exception: # nosec # pylint: disable=broad-except
|
81
81
|
pass
|
82
82
|
|
83
|
-
|
83
|
+
_LOG.debug("checking memory growth for %s", path)
|
84
84
|
|
85
85
|
peak_stats: dict[Any, Any] = {}
|
86
86
|
for i in range(3):
|
@@ -91,10 +91,10 @@ def _dump_memory_diff(request: pyramid.request.Request) -> list[Any]:
|
|
91
91
|
response = None
|
92
92
|
try:
|
93
93
|
response = request.invoke_subrequest(sub_request)
|
94
|
-
|
94
|
+
_LOG.debug("response was %d", response.status_code)
|
95
95
|
|
96
96
|
except HTTPException as ex:
|
97
|
-
|
97
|
+
_LOG.debug("response was %s", str(ex))
|
98
98
|
|
99
99
|
del response
|
100
100
|
|
@@ -142,6 +142,7 @@ def _error(request: pyramid.request.Request) -> Any:
|
|
142
142
|
|
143
143
|
|
144
144
|
def _time(request: pyramid.request.Request) -> Any:
|
145
|
+
del request # unused
|
145
146
|
return {
|
146
147
|
"local_time": str(datetime.now()),
|
147
148
|
"gmt_time": str(datetime.utcnow()),
|
@@ -188,10 +189,17 @@ def _show_refs(request: pyramid.request.Request) -> pyramid.response.Response:
|
|
188
189
|
args["extra_info"] = lambda obj: f"{get_size(obj) / 1024:.3f} kb\n{id(obj)}"
|
189
190
|
|
190
191
|
result = StringIO()
|
191
|
-
if request.params.get("backrefs", "")
|
192
|
-
|
193
|
-
|
194
|
-
|
192
|
+
if request.params.get("backrefs", "") == "":
|
193
|
+
|
194
|
+
def new_filter(x: Any) -> bool:
|
195
|
+
return not objgraph.inspect.isclass(x)
|
196
|
+
|
197
|
+
if "filter" in args:
|
198
|
+
old_filter = args["filter"]
|
199
|
+
args["filter"] = lambda x: old_filter(x) and new_filter(x)
|
200
|
+
else:
|
201
|
+
args["filter"] = new_filter
|
202
|
+
objgraph.show_backrefs(objs, output=result, **args)
|
195
203
|
|
196
204
|
request.response.content_type = "text/vnd.graphviz"
|
197
205
|
request.response.text = result.getvalue()
|
@@ -211,4 +219,4 @@ def init(config: pyramid.config.Configurator) -> None:
|
|
211
219
|
_add_view(config, "error", "error", _error)
|
212
220
|
_add_view(config, "time", "time", _time)
|
213
221
|
_add_view(config, "show_refs", "show_refs.dot", _show_refs)
|
214
|
-
|
222
|
+
_LOG.info("Enabled the /debug/... API")
|
c2cwsgiutils/debug/utils.py
CHANGED
@@ -8,18 +8,18 @@ from types import FunctionType, ModuleType
|
|
8
8
|
from typing import Any
|
9
9
|
|
10
10
|
# 7ff7d33bd000-7ff7d33be000 r--p 00000000 00:65 49 /usr/lib/toto.so
|
11
|
-
|
11
|
+
_SMAPS_LOCATION_RE = re.compile(r"^[0-9a-f]+-[0-9a-f]+ +.... +[0-9a-f]+ +[^ ]+ +\d+ +(.*)$")
|
12
12
|
|
13
13
|
# Size: 4 kB
|
14
|
-
|
14
|
+
_SMAPS_ENTRY_RE = re.compile(r"^([\w]+): +(\d+) kB$")
|
15
15
|
|
16
|
-
|
17
|
-
|
16
|
+
_BLACKLIST = type, ModuleType, FunctionType
|
17
|
+
_LOG = logging.getLogger(__name__)
|
18
18
|
|
19
19
|
|
20
20
|
def get_size(obj: Any) -> int:
|
21
21
|
"""Get the sum size of object & members."""
|
22
|
-
if isinstance(obj,
|
22
|
+
if isinstance(obj, _BLACKLIST):
|
23
23
|
return 0
|
24
24
|
seen_ids: set[int] = set()
|
25
25
|
size = 0
|
@@ -27,7 +27,7 @@ def get_size(obj: Any) -> int:
|
|
27
27
|
while objects:
|
28
28
|
need_referents = []
|
29
29
|
for obj_ in objects:
|
30
|
-
if not isinstance(obj_,
|
30
|
+
if not isinstance(obj_, _BLACKLIST) and id(obj_) not in seen_ids:
|
31
31
|
seen_ids.add(id(obj_))
|
32
32
|
size += sys.getsizeof(obj_)
|
33
33
|
need_referents.append(obj_)
|
@@ -45,11 +45,11 @@ def dump_memory_maps(pid: str = "self") -> list[dict[str, Any]]:
|
|
45
45
|
sizes: dict[str, Any] = {}
|
46
46
|
for line in input_:
|
47
47
|
line = line.rstrip("\n")
|
48
|
-
matcher =
|
48
|
+
matcher = _SMAPS_LOCATION_RE.match(line)
|
49
49
|
if matcher:
|
50
50
|
cur_dict = sizes.setdefault(matcher.group(1), defaultdict(int))
|
51
51
|
else:
|
52
|
-
matcher =
|
52
|
+
matcher = _SMAPS_ENTRY_RE.match(line)
|
53
53
|
if matcher:
|
54
54
|
name = matcher.group(1)
|
55
55
|
if name in ("Size", "Rss", "Pss"):
|
@@ -59,5 +59,5 @@ def dump_memory_maps(pid: str = "self") -> list[dict[str, Any]]:
|
|
59
59
|
and not line.startswith("ProtectionKey:")
|
60
60
|
and not line.startswith("THPeligible:")
|
61
61
|
):
|
62
|
-
|
62
|
+
_LOG.debug("Don't know how to parse /proc/%s/smaps line: %s", pid, line)
|
63
63
|
return [{"name": name, **value} for name, value in sizes.items() if value.get("pss_kb", 0) > 0]
|
c2cwsgiutils/errors.py
CHANGED
@@ -14,14 +14,12 @@ from webob.request import DisconnectionError
|
|
14
14
|
|
15
15
|
from c2cwsgiutils import auth, config_utils
|
16
16
|
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
401: LOG.debug,
|
24
|
-
500: LOG.error,
|
17
|
+
_DEVELOPMENT = os.environ.get("DEVELOPMENT", "0") != "0"
|
18
|
+
|
19
|
+
_LOG = logging.getLogger(__name__)
|
20
|
+
_STATUS_LOGGER = {
|
21
|
+
401: _LOG.debug,
|
22
|
+
500: _LOG.error,
|
25
23
|
# The rest are warnings
|
26
24
|
}
|
27
25
|
|
@@ -53,7 +51,7 @@ def _do_error(
|
|
53
51
|
request: pyramid.request.Request,
|
54
52
|
status: int,
|
55
53
|
exception: Exception,
|
56
|
-
logger: Callable[..., None] =
|
54
|
+
logger: Callable[..., None] = _LOG.error,
|
57
55
|
reduce_info_sent: Callable[[Exception], None] = lambda e: None,
|
58
56
|
) -> pyramid.response.Response:
|
59
57
|
logger(
|
@@ -82,7 +80,7 @@ def _do_error(
|
|
82
80
|
|
83
81
|
def _http_error(exception: HTTPException, request: pyramid.request.Request) -> Any:
|
84
82
|
if request.method != "OPTIONS":
|
85
|
-
log =
|
83
|
+
log = _STATUS_LOGGER.get(exception.status_code, _LOG.warning)
|
86
84
|
log(
|
87
85
|
"%s %s returned status code %s: %s",
|
88
86
|
request.method,
|
@@ -102,7 +100,7 @@ def _http_error(exception: HTTPException, request: pyramid.request.Request) -> A
|
|
102
100
|
|
103
101
|
|
104
102
|
def _include_dev_details(request: pyramid.request.Request) -> bool:
|
105
|
-
return
|
103
|
+
return _DEVELOPMENT or auth.is_auth(request)
|
106
104
|
|
107
105
|
|
108
106
|
def _integrity_error(
|
@@ -121,7 +119,7 @@ def _client_interrupted_error(
|
|
121
119
|
exception: Exception, request: pyramid.request.Request
|
122
120
|
) -> pyramid.response.Response:
|
123
121
|
# No need to cry wolf if it's just the client that interrupted the connection
|
124
|
-
return _do_error(request, 500, exception, logger=
|
122
|
+
return _do_error(request, 500, exception, logger=_LOG.info)
|
125
123
|
|
126
124
|
|
127
125
|
def _boto_client_error(exception: Any, request: pyramid.request.Request) -> pyramid.response.Response:
|
@@ -132,7 +130,7 @@ def _boto_client_error(exception: Any, request: pyramid.request.Request) -> pyra
|
|
132
130
|
status_code = exception.response["ResponseMetadata"]["HTTPStatusCode"]
|
133
131
|
else:
|
134
132
|
status_code = int(exception.response["Error"]["Code"])
|
135
|
-
log =
|
133
|
+
log = _STATUS_LOGGER.get(status_code, _LOG.warning)
|
136
134
|
return _do_error(request, status_code, exception, logger=log)
|
137
135
|
|
138
136
|
|
@@ -143,7 +141,7 @@ def _other_error(exception: Exception, request: pyramid.request.Request) -> pyra
|
|
143
141
|
status = 500
|
144
142
|
if exception_class == "beaker.exceptions.BeakerException" and str(exception) == "Invalid signature":
|
145
143
|
status = 401
|
146
|
-
|
144
|
+
_LOG.debug("Actual exception: %s.%s", exception.__class__.__module__, exception.__class__.__name__)
|
147
145
|
return _do_error(request, status, exception)
|
148
146
|
|
149
147
|
|
@@ -179,4 +177,4 @@ def includeme(config: pyramid.config.Configurator) -> None:
|
|
179
177
|
config.add_view(view=_client_interrupted_error, context=exception, **common_options)
|
180
178
|
|
181
179
|
config.add_view(view=_other_error, context=Exception, **common_options)
|
182
|
-
|
180
|
+
_LOG.info("Installed the error catching views")
|