c2cwsgiutils 6.2.0.dev74__py3-none-any.whl → 6.2.0.dev75__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- c2cwsgiutils/__init__.py +4 -4
- c2cwsgiutils/acceptance/__init__.py +4 -1
- c2cwsgiutils/acceptance/connection.py +45 -32
- c2cwsgiutils/acceptance/image.py +45 -39
- c2cwsgiutils/acceptance/print.py +2 -2
- c2cwsgiutils/acceptance/utils.py +4 -4
- c2cwsgiutils/auth.py +20 -13
- c2cwsgiutils/broadcast/__init__.py +25 -16
- c2cwsgiutils/broadcast/interface.py +8 -4
- c2cwsgiutils/broadcast/local.py +9 -4
- c2cwsgiutils/broadcast/redis.py +19 -12
- c2cwsgiutils/client_info.py +3 -2
- c2cwsgiutils/config_utils.py +14 -10
- c2cwsgiutils/coverage_setup.py +5 -5
- c2cwsgiutils/db.py +54 -47
- c2cwsgiutils/db_maintenance_view.py +7 -8
- c2cwsgiutils/debug/__init__.py +1 -2
- c2cwsgiutils/debug/_listeners.py +6 -4
- c2cwsgiutils/debug/_views.py +15 -10
- c2cwsgiutils/debug/utils.py +5 -5
- c2cwsgiutils/errors.py +15 -10
- c2cwsgiutils/health_check.py +79 -60
- c2cwsgiutils/index.py +35 -37
- c2cwsgiutils/loader.py +5 -4
- c2cwsgiutils/logging_view.py +13 -6
- c2cwsgiutils/models_graph.py +5 -5
- c2cwsgiutils/pretty_json.py +3 -2
- c2cwsgiutils/profiler.py +1 -2
- c2cwsgiutils/prometheus.py +6 -6
- c2cwsgiutils/pyramid.py +1 -1
- c2cwsgiutils/pyramid_logging.py +9 -4
- c2cwsgiutils/redis_stats.py +12 -7
- c2cwsgiutils/redis_utils.py +18 -10
- c2cwsgiutils/request_tracking/__init__.py +20 -12
- c2cwsgiutils/request_tracking/_sql.py +2 -1
- c2cwsgiutils/scripts/genversion.py +10 -9
- c2cwsgiutils/scripts/stats_db.py +29 -13
- c2cwsgiutils/sentry.py +44 -20
- c2cwsgiutils/setup_process.py +7 -4
- c2cwsgiutils/sql_profiler/_impl.py +12 -11
- c2cwsgiutils/sqlalchemylogger/_models.py +3 -3
- c2cwsgiutils/sqlalchemylogger/examples/__init__.py +0 -0
- c2cwsgiutils/sqlalchemylogger/handlers.py +8 -9
- c2cwsgiutils/stats_pyramid/_db_spy.py +12 -3
- c2cwsgiutils/stats_pyramid/_pyramid_spy.py +10 -9
- c2cwsgiutils/version.py +10 -7
- {c2cwsgiutils-6.2.0.dev74.dist-info → c2cwsgiutils-6.2.0.dev75.dist-info}/METADATA +1 -1
- c2cwsgiutils-6.2.0.dev75.dist-info/RECORD +68 -0
- c2cwsgiutils-6.2.0.dev74.dist-info/RECORD +0 -67
- {c2cwsgiutils-6.2.0.dev74.dist-info → c2cwsgiutils-6.2.0.dev75.dist-info}/LICENSE +0 -0
- {c2cwsgiutils-6.2.0.dev74.dist-info → c2cwsgiutils-6.2.0.dev75.dist-info}/WHEEL +0 -0
- {c2cwsgiutils-6.2.0.dev74.dist-info → c2cwsgiutils-6.2.0.dev75.dist-info}/entry_points.txt +0 -0
c2cwsgiutils/index.py
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
import logging
|
2
2
|
import urllib.parse
|
3
3
|
import warnings
|
4
|
-
from typing import Any,
|
4
|
+
from typing import Any, cast
|
5
5
|
|
6
6
|
import jwt
|
7
7
|
import pyramid.config
|
@@ -44,22 +44,20 @@ from c2cwsgiutils.config_utils import env_or_settings
|
|
44
44
|
|
45
45
|
_LOG = logging.getLogger(__name__)
|
46
46
|
|
47
|
-
additional_title:
|
47
|
+
additional_title: str | None = None
|
48
48
|
additional_noauth: list[str] = []
|
49
49
|
additional_auth: list[str] = []
|
50
50
|
_ELEM_ID = 0
|
51
51
|
|
52
52
|
|
53
|
-
def _url(
|
54
|
-
request: pyramid.request.Request, route: str, params: Optional[dict[str, str]] = None
|
55
|
-
) -> Optional[str]:
|
53
|
+
def _url(request: pyramid.request.Request, route: str, params: dict[str, str] | None = None) -> str | None:
|
56
54
|
try:
|
57
|
-
return request.route_url(route, _query=params) # type: ignore
|
55
|
+
return request.route_url(route, _query=params) # type: ignore[no-any-return]
|
58
56
|
except KeyError:
|
59
57
|
return None
|
60
58
|
|
61
59
|
|
62
|
-
def section(title: str, *content: str, sep:
|
60
|
+
def section(title: str, *content: str, sep: bool | None = True) -> str:
|
63
61
|
"""Get an HTML section."""
|
64
62
|
printable_content = "\n".join(content)
|
65
63
|
result = f"""
|
@@ -73,7 +71,7 @@ def section(title: str, *content: str, sep: Optional[bool] = True) -> str:
|
|
73
71
|
return result
|
74
72
|
|
75
73
|
|
76
|
-
def paragraph(*content: str, title:
|
74
|
+
def paragraph(*content: str, title: str | None = None) -> str:
|
77
75
|
"""Get an HTML paragraph."""
|
78
76
|
body = ""
|
79
77
|
if title:
|
@@ -82,7 +80,7 @@ def paragraph(*content: str, title: Optional[str] = None) -> str:
|
|
82
80
|
return "<p>" + body + "</p>"
|
83
81
|
|
84
82
|
|
85
|
-
def link(url:
|
83
|
+
def link(url: str | None, label: str, cssclass: str = "btn btn-primary", target: str = "_blank") -> str:
|
86
84
|
"""Get an HTML link."""
|
87
85
|
attrs = ""
|
88
86
|
if cssclass:
|
@@ -91,11 +89,10 @@ def link(url: Optional[str], label: str, cssclass: str = "btn btn-primary", targ
|
|
91
89
|
attrs += f' target="{target}"'
|
92
90
|
if url is not None:
|
93
91
|
return f'<a href="{url}"{attrs}>{label}</a>'
|
94
|
-
|
95
|
-
return ""
|
92
|
+
return ""
|
96
93
|
|
97
94
|
|
98
|
-
def form(url:
|
95
|
+
def form(url: str | None, *content: str, method: str = "get", target: str = "_blank") -> str:
|
99
96
|
"""Get an HTML form."""
|
100
97
|
assert url is not None
|
101
98
|
method_attrs = ""
|
@@ -109,9 +106,7 @@ def form(url: Optional[str], *content: str, method: str = "get", target: str = "
|
|
109
106
|
"""
|
110
107
|
|
111
108
|
|
112
|
-
def input_(
|
113
|
-
name: str, label: Optional[str] = None, type_: Optional[str] = None, value: Union[str, int] = ""
|
114
|
-
) -> str:
|
109
|
+
def input_(name: str, label: str | None = None, type_: str | None = None, value: str | int = "") -> str:
|
115
110
|
"""Get an HTML input."""
|
116
111
|
global _ELEM_ID # pylint: disable=global-statement
|
117
112
|
id_ = _ELEM_ID
|
@@ -214,16 +209,14 @@ def _versions(request: pyramid.request.Request) -> str:
|
|
214
209
|
versions_url = _url(request, "c2c_versions")
|
215
210
|
if versions_url:
|
216
211
|
return section("Versions " + link(versions_url, "Get"), sep=False)
|
217
|
-
|
218
|
-
return ""
|
212
|
+
return ""
|
219
213
|
|
220
214
|
|
221
215
|
def _stats(request: pyramid.request.Request) -> str:
|
222
216
|
stats_url = _url(request, "c2c_read_stats_json")
|
223
217
|
if stats_url:
|
224
218
|
return section("Statistics", paragraph(link(stats_url, "Get")), sep=False)
|
225
|
-
|
226
|
-
return ""
|
219
|
+
return ""
|
227
220
|
|
228
221
|
|
229
222
|
def _profiler(request: pyramid.request.Request) -> str:
|
@@ -236,12 +229,11 @@ def _profiler(request: pyramid.request.Request) -> str:
|
|
236
229
|
link(sql_profiler_url, "Status"),
|
237
230
|
link(sql_profiler_url + "?enable=1", "Enable"),
|
238
231
|
link(sql_profiler_url + "?enable=0", "Disable"),
|
239
|
-
]
|
232
|
+
],
|
240
233
|
),
|
241
234
|
sep=False,
|
242
235
|
)
|
243
|
-
|
244
|
-
return ""
|
236
|
+
return ""
|
245
237
|
|
246
238
|
|
247
239
|
def _db_maintenance(request: pyramid.request.Request) -> str:
|
@@ -262,8 +254,7 @@ def _db_maintenance(request: pyramid.request.Request) -> str:
|
|
262
254
|
),
|
263
255
|
sep=False,
|
264
256
|
)
|
265
|
-
|
266
|
-
return ""
|
257
|
+
return ""
|
267
258
|
|
268
259
|
|
269
260
|
def _logging(request: pyramid.request.Request) -> str:
|
@@ -285,8 +276,7 @@ def _logging(request: pyramid.request.Request) -> str:
|
|
285
276
|
paragraph(link(logging_url, "List overrides")),
|
286
277
|
sep=False,
|
287
278
|
)
|
288
|
-
|
289
|
-
return ""
|
279
|
+
return ""
|
290
280
|
|
291
281
|
|
292
282
|
def _debug(request: pyramid.request.Request) -> str:
|
@@ -300,7 +290,7 @@ def _debug(request: pyramid.request.Request) -> str:
|
|
300
290
|
link(_url(request, "c2c_debug_stacks"), "Stack traces"),
|
301
291
|
link(_url(request, "c2c_debug_headers"), "HTTP headers"),
|
302
292
|
link(_url(request, "c2c_debug_memory_maps"), "Mapped memory"),
|
303
|
-
]
|
293
|
+
],
|
304
294
|
),
|
305
295
|
'<h2>Memory usage<span style="font-size: 0.5em;">, with <a href="https://mg.pov.lt/objgraph/">objgraph</a></span></h2>',
|
306
296
|
"<p>Runs the garbage collector and dumps the memory usage as JSON.</p>",
|
@@ -345,8 +335,7 @@ def _debug(request: pyramid.request.Request) -> str:
|
|
345
335
|
),
|
346
336
|
sep=False,
|
347
337
|
)
|
348
|
-
|
349
|
-
return ""
|
338
|
+
return ""
|
350
339
|
|
351
340
|
|
352
341
|
def _health_check(request: pyramid.request.Request) -> str:
|
@@ -362,8 +351,7 @@ def _health_check(request: pyramid.request.Request) -> str:
|
|
362
351
|
),
|
363
352
|
sep=False,
|
364
353
|
)
|
365
|
-
|
366
|
-
return ""
|
354
|
+
return ""
|
367
355
|
|
368
356
|
|
369
357
|
def _github_login(request: pyramid.request.Request) -> dict[str, Any]:
|
@@ -391,7 +379,10 @@ def _github_login(request: pyramid.request.Request) -> dict[str, Any]:
|
|
391
379
|
)
|
392
380
|
authorization_url, state = oauth.authorization_url(
|
393
381
|
env_or_settings(
|
394
|
-
settings,
|
382
|
+
settings,
|
383
|
+
GITHUB_AUTH_URL_ENV,
|
384
|
+
GITHUB_AUTH_URL_PROP,
|
385
|
+
"https://github.com/login/oauth/authorize",
|
395
386
|
),
|
396
387
|
)
|
397
388
|
use_session = env_or_settings(settings, USE_SESSION_ENV, USE_SESSION_PROP, "").lower() == "true"
|
@@ -451,7 +442,7 @@ def _github_login_callback(request: pyramid.request.Request) -> dict[str, Any]:
|
|
451
442
|
GITHUB_USER_URL_ENV,
|
452
443
|
GITHUB_USER_URL_PROP,
|
453
444
|
"https://api.github.com/user",
|
454
|
-
)
|
445
|
+
),
|
455
446
|
).json()
|
456
447
|
|
457
448
|
user_information: UserDetails = {
|
@@ -495,7 +486,8 @@ def _github_logout(request: pyramid.request.Request) -> dict[str, Any]:
|
|
495
486
|
),
|
496
487
|
)
|
497
488
|
raise HTTPFound(
|
498
|
-
location=request.params.get("came_from", _url(request, "c2c_index")),
|
489
|
+
location=request.params.get("came_from", _url(request, "c2c_index")),
|
490
|
+
headers=request.response.headers,
|
499
491
|
)
|
500
492
|
|
501
493
|
|
@@ -515,7 +507,10 @@ def includeme(config: pyramid.config.Configurator) -> None:
|
|
515
507
|
config.add_view(_index, route_name="c2c_index", http_cache=0, renderer="./templates/index.html.mako")
|
516
508
|
config.add_route("c2c_index_slash", base_path + "/", request_method=("GET", "POST"))
|
517
509
|
config.add_view(
|
518
|
-
_index,
|
510
|
+
_index,
|
511
|
+
route_name="c2c_index_slash",
|
512
|
+
http_cache=0,
|
513
|
+
renderer="./templates/index.html.mako",
|
519
514
|
)
|
520
515
|
|
521
516
|
settings = config.get_settings()
|
@@ -523,7 +518,7 @@ def includeme(config: pyramid.config.Configurator) -> None:
|
|
523
518
|
if auth_type_ == AuthenticationType.SECRET:
|
524
519
|
_LOG.warning(
|
525
520
|
"It is recommended to use OAuth2 with GitHub login instead of the `C2C_SECRET` because it "
|
526
|
-
"protects from brute force attacks and the access grant is personal and can be revoked."
|
521
|
+
"protects from brute force attacks and the access grant is personal and can be revoked.",
|
527
522
|
)
|
528
523
|
|
529
524
|
if auth_type_ == AuthenticationType.GITHUB:
|
@@ -531,7 +526,10 @@ def includeme(config: pyramid.config.Configurator) -> None:
|
|
531
526
|
config.add_view(_github_login, route_name="c2c_github_login", http_cache=0)
|
532
527
|
config.add_route("c2c_github_callback", base_path + "/github-callback", request_method=("GET",))
|
533
528
|
config.add_view(
|
534
|
-
_github_login_callback,
|
529
|
+
_github_login_callback,
|
530
|
+
route_name="c2c_github_callback",
|
531
|
+
http_cache=0,
|
532
|
+
renderer="fast_json",
|
535
533
|
)
|
536
534
|
config.add_route("c2c_github_logout", base_path + "/github-logout", request_method=("GET",))
|
537
535
|
config.add_view(_github_logout, route_name="c2c_github_logout", http_cache=0)
|
c2cwsgiutils/loader.py
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
import logging.config
|
2
|
-
from typing import
|
2
|
+
from typing import cast
|
3
3
|
|
4
4
|
from plaster_pastedeploy import Loader as BaseLoader
|
5
5
|
|
@@ -8,10 +8,10 @@ from c2cwsgiutils import get_config_defaults, get_logconfig_dict
|
|
8
8
|
_LOG = logging.getLogger(__name__)
|
9
9
|
|
10
10
|
|
11
|
-
class Loader(BaseLoader): # type: ignore
|
11
|
+
class Loader(BaseLoader): # type: ignore[misc]
|
12
12
|
"""The application loader."""
|
13
13
|
|
14
|
-
def _get_defaults(self, defaults:
|
14
|
+
def _get_defaults(self, defaults: dict[str, str] | None = None) -> dict[str, str]:
|
15
15
|
d = get_config_defaults()
|
16
16
|
d.update(defaults or {})
|
17
17
|
return cast(dict[str, str], super()._get_defaults(d))
|
@@ -20,7 +20,7 @@ class Loader(BaseLoader): # type: ignore
|
|
20
20
|
"""Get the object representation."""
|
21
21
|
return f'c2cwsgiutils.loader.Loader(uri="{self.uri}")'
|
22
22
|
|
23
|
-
def setup_logging(self, defaults:
|
23
|
+
def setup_logging(self, defaults: dict[str, str] | None = None) -> None:
|
24
24
|
"""
|
25
25
|
Set up logging via :func:`logging.config.dictConfig` with value returned from c2cwsgiutils.get_logconfig_dict.
|
26
26
|
|
@@ -34,6 +34,7 @@ class Loader(BaseLoader): # type: ignore
|
|
34
34
|
:func:`logging.config.fileConfig`.
|
35
35
|
|
36
36
|
"""
|
37
|
+
del defaults # Unused
|
37
38
|
if "loggers" in self.get_sections():
|
38
39
|
logging.config.dictConfig(get_logconfig_dict(self.uri.path))
|
39
40
|
else:
|
c2cwsgiutils/logging_view.py
CHANGED
@@ -23,10 +23,15 @@ def includeme(config: pyramid.config.Configurator) -> None:
|
|
23
23
|
"""Install the view to configure the loggers, if configured to do so."""
|
24
24
|
if auth.is_enabled(config, _ENV_KEY, _CONFIG_KEY):
|
25
25
|
config.add_route(
|
26
|
-
"c2c_logging_level",
|
26
|
+
"c2c_logging_level",
|
27
|
+
config_utils.get_base_path(config) + r"/logging/level",
|
28
|
+
request_method="GET",
|
27
29
|
)
|
28
30
|
config.add_view(
|
29
|
-
_logging_change_level,
|
31
|
+
_logging_change_level,
|
32
|
+
route_name="c2c_logging_level",
|
33
|
+
renderer="fast_json",
|
34
|
+
http_cache=0,
|
30
35
|
)
|
31
36
|
_restore_overrides(config)
|
32
37
|
_LOG.info("Enabled the /logging/level API")
|
@@ -40,7 +45,10 @@ def _logging_change_level(request: pyramid.request.Request) -> Mapping[str, Any]
|
|
40
45
|
logger = logging.getLogger(name)
|
41
46
|
if level is not None:
|
42
47
|
_LOG.critical(
|
43
|
-
"Logging of %s changed from %s to %s",
|
48
|
+
"Logging of %s changed from %s to %s",
|
49
|
+
name,
|
50
|
+
logging.getLevelName(logger.level),
|
51
|
+
level,
|
44
52
|
)
|
45
53
|
_set_level(name=name, level=level)
|
46
54
|
_store_override(request.registry.settings, name, level)
|
@@ -50,8 +58,7 @@ def _logging_change_level(request: pyramid.request.Request) -> Mapping[str, Any]
|
|
50
58
|
"level": logging.getLevelName(logger.level),
|
51
59
|
"effective_level": logging.getLevelName(logger.getEffectiveLevel()),
|
52
60
|
}
|
53
|
-
|
54
|
-
return {"status": 200, "overrides": dict(_list_overrides(request.registry.settings))}
|
61
|
+
return {"status": 200, "overrides": dict(_list_overrides(request.registry.settings))}
|
55
62
|
|
56
63
|
|
57
64
|
@broadcast.decorator(expect_answers=True)
|
@@ -67,7 +74,7 @@ def _restore_overrides(config: pyramid.config.Configurator) -> None:
|
|
67
74
|
logging.getLogger(name).setLevel(level)
|
68
75
|
except ImportError:
|
69
76
|
pass # don't have redis
|
70
|
-
except Exception: # pylint: disable=broad-
|
77
|
+
except Exception: # pylint: disable=broad-exception-caught
|
71
78
|
# survive an error there. Logging levels is not business critical...
|
72
79
|
_LOG.warning("Cannot restore logging levels", exc_info=True)
|
73
80
|
|
c2cwsgiutils/models_graph.py
CHANGED
@@ -24,7 +24,7 @@ def _generate_model_graph(module: Any, base: Any) -> None:
|
|
24
24
|
"""
|
25
25
|
digraph {
|
26
26
|
rankdir=BT;
|
27
|
-
"""
|
27
|
+
""",
|
28
28
|
)
|
29
29
|
|
30
30
|
interesting = {
|
@@ -34,7 +34,7 @@ def _generate_model_graph(module: Any, base: Any) -> None:
|
|
34
34
|
}
|
35
35
|
|
36
36
|
for symbol in list(interesting):
|
37
|
-
symbol = getattr(module, symbol.__name__)
|
37
|
+
symbol = getattr(module, symbol.__name__) # noqa: PLW2901
|
38
38
|
if _is_interesting(symbol, base):
|
39
39
|
_print_node(symbol, interesting)
|
40
40
|
|
@@ -56,7 +56,7 @@ def _is_interesting(what: Any, base: type) -> bool:
|
|
56
56
|
|
57
57
|
|
58
58
|
def _get_table_desc(symbol: Any) -> str:
|
59
|
-
cols = [symbol.__name__, ""
|
59
|
+
cols = [symbol.__name__, "", *_get_local_cols(symbol)]
|
60
60
|
|
61
61
|
return "\\n".join(cols)
|
62
62
|
|
@@ -70,7 +70,7 @@ def _get_all_cols(symbol: Any) -> list[str]:
|
|
70
70
|
# Those are not fields
|
71
71
|
pass
|
72
72
|
elif isinstance(member, sa.sql.schema.SchemaItem):
|
73
|
-
cols.append(member_name + ("[null]" if member.nullable else "")) # type: ignore
|
73
|
+
cols.append(member_name + ("[null]" if member.nullable else "")) # type: ignore[attr-defined]
|
74
74
|
elif isinstance(member, sa.orm.attributes.InstrumentedAttribute):
|
75
75
|
nullable = (
|
76
76
|
member.property.columns[0].nullable
|
@@ -88,4 +88,4 @@ def _get_local_cols(symbol: Any) -> list[str]:
|
|
88
88
|
for parent in symbol.__bases__:
|
89
89
|
result -= set(_get_all_cols(parent))
|
90
90
|
|
91
|
-
return sorted(
|
91
|
+
return sorted(result)
|
c2cwsgiutils/pretty_json.py
CHANGED
@@ -35,7 +35,7 @@ def init(config: pyramid.config.Configurator) -> None:
|
|
35
35
|
def includeme(config: pyramid.config.Configurator) -> None:
|
36
36
|
"""Initialize json and fast_json renderer."""
|
37
37
|
pretty_print = config_bool(
|
38
|
-
env_or_config(config, "C2C_JSON_PRETTY_PRINT", "c2c.json.pretty_print", "false")
|
38
|
+
env_or_config(config, "C2C_JSON_PRETTY_PRINT", "c2c.json.pretty_print", "false"),
|
39
39
|
)
|
40
40
|
sort_keys = config_bool(env_or_config(config, "C2C_JSON_SORT_KEYS", "c2c.json.sort_keys", "false"))
|
41
41
|
|
@@ -44,6 +44,7 @@ def includeme(config: pyramid.config.Configurator) -> None:
|
|
44
44
|
config.add_renderer("json", JSON(indent=2 if pretty_print else None, sort_keys=sort_keys))
|
45
45
|
config.add_renderer("fast_json", JSON(serializer=fast_dump))
|
46
46
|
config.add_renderer(
|
47
|
-
"cornice_json",
|
47
|
+
"cornice_json",
|
48
|
+
CorniceRenderer(indent=2 if pretty_print else None, sort_keys=sort_keys),
|
48
49
|
)
|
49
50
|
config.add_renderer("cornice_fast_json", CorniceRenderer(serializer=fast_dump))
|
c2cwsgiutils/profiler.py
CHANGED
@@ -2,7 +2,6 @@ import contextlib
|
|
2
2
|
import cProfile
|
3
3
|
import pstats
|
4
4
|
import sys
|
5
|
-
from typing import Any
|
6
5
|
|
7
6
|
|
8
7
|
class Profile(contextlib.ContextDecorator):
|
@@ -17,7 +16,7 @@ class Profile(contextlib.ContextDecorator):
|
|
17
16
|
def __enter__(self) -> None:
|
18
17
|
self.pr.enable()
|
19
18
|
|
20
|
-
def __exit__(self, *exc:
|
19
|
+
def __exit__(self, *exc: object) -> None:
|
21
20
|
del exc
|
22
21
|
|
23
22
|
self.pr.disable()
|
c2cwsgiutils/prometheus.py
CHANGED
@@ -4,7 +4,7 @@ import os
|
|
4
4
|
import re
|
5
5
|
import resource
|
6
6
|
from collections.abc import Generator, Iterable
|
7
|
-
from typing import Any,
|
7
|
+
from typing import Any, TypedDict, cast
|
8
8
|
|
9
9
|
import prometheus_client
|
10
10
|
import prometheus_client.core
|
@@ -37,7 +37,7 @@ def start_single_process() -> None:
|
|
37
37
|
prometheus_client.start_http_server(int(os.environ["C2C_PROMETHEUS_PORT"]))
|
38
38
|
|
39
39
|
|
40
|
-
def start(registry:
|
40
|
+
def start(registry: prometheus_client.CollectorRegistry | None = None) -> None:
|
41
41
|
"""Start separate HTTP server to provide the Prometheus metrics."""
|
42
42
|
if os.environ.get("C2C_PROMETHEUS_PORT") is not None:
|
43
43
|
broadcast.includeme()
|
@@ -114,7 +114,7 @@ def serialize_collected_data(collector: prometheus_client.registry.Collector) ->
|
|
114
114
|
elif isinstance(process_gauge, prometheus_client.core.CounterMetricFamily):
|
115
115
|
gauge["type"] = "counter"
|
116
116
|
else:
|
117
|
-
raise NotImplementedError
|
117
|
+
raise NotImplementedError
|
118
118
|
for sample in process_gauge.samples:
|
119
119
|
gauge["samples"].append(
|
120
120
|
{
|
@@ -153,12 +153,12 @@ def _deserialize_collected_data(
|
|
153
153
|
|
154
154
|
if serialized_metric["type"] == "gauge":
|
155
155
|
metric: prometheus_client.core.Metric = prometheus_client.core.GaugeMetricFamily(
|
156
|
-
**serialized_metric["args"]
|
156
|
+
**serialized_metric["args"],
|
157
157
|
)
|
158
158
|
elif serialized_metric["type"] == "counter":
|
159
159
|
metric = prometheus_client.core.CounterMetricFamily(**serialized_metric["args"])
|
160
160
|
else:
|
161
|
-
raise NotImplementedError
|
161
|
+
raise NotImplementedError
|
162
162
|
for sample in serialized_metric["samples"]:
|
163
163
|
metric.samples.append(
|
164
164
|
prometheus_client.metrics_core.Sample(**sample), # type: ignore[attr-defined]
|
@@ -169,7 +169,7 @@ def _deserialize_collected_data(
|
|
169
169
|
class MemoryMapCollector(prometheus_client.registry.Collector):
|
170
170
|
"""The Linux memory map provider."""
|
171
171
|
|
172
|
-
def __init__(self, memory_type: str = "pss", pids:
|
172
|
+
def __init__(self, memory_type: str = "pss", pids: list[str] | None = None) -> None:
|
173
173
|
"""
|
174
174
|
Initialize.
|
175
175
|
|
c2cwsgiutils/pyramid.py
CHANGED
@@ -33,7 +33,7 @@ def includeme(config: pyramid.config.Configurator) -> None:
|
|
33
33
|
config: The pyramid Configuration
|
34
34
|
|
35
35
|
"""
|
36
|
-
logging.captureWarnings(True)
|
36
|
+
logging.captureWarnings(capture=True)
|
37
37
|
config.include(coverage_setup.includeme)
|
38
38
|
config.include(sentry.includeme)
|
39
39
|
config.add_settings(handle_exceptions=False)
|
c2cwsgiutils/pyramid_logging.py
CHANGED
@@ -16,7 +16,7 @@ import logging
|
|
16
16
|
import logging.config
|
17
17
|
import socket
|
18
18
|
from collections.abc import Mapping, MutableMapping
|
19
|
-
from typing import TYPE_CHECKING, Any,
|
19
|
+
from typing import TYPE_CHECKING, Any, TextIO
|
20
20
|
|
21
21
|
import cee_syslog_handler
|
22
22
|
from pyramid.threadlocal import get_current_request
|
@@ -83,7 +83,7 @@ def _make_message_dict(*args: Any, **kargv: Any) -> Mapping[str, Any]:
|
|
83
83
|
return _un_underscore(msg)
|
84
84
|
|
85
85
|
|
86
|
-
class PyramidCeeSysLogHandler(cee_syslog_handler.CeeSysLogHandler): # type: ignore
|
86
|
+
class PyramidCeeSysLogHandler(cee_syslog_handler.CeeSysLogHandler): # type: ignore[misc]
|
87
87
|
"""A CEE (JSON format) log handler with additional information about the current request."""
|
88
88
|
|
89
89
|
def __init__(self, *args: Any, **kargv: Any) -> None:
|
@@ -113,7 +113,7 @@ else:
|
|
113
113
|
class JsonLogHandler(Base):
|
114
114
|
"""Log to stdout in JSON."""
|
115
115
|
|
116
|
-
def __init__(self, stream:
|
116
|
+
def __init__(self, stream: TextIO | None = None) -> None:
|
117
117
|
"""Initialize the handler."""
|
118
118
|
super().__init__(stream)
|
119
119
|
self.addFilter(_PYRAMID_FILTER)
|
@@ -122,6 +122,11 @@ class JsonLogHandler(Base):
|
|
122
122
|
def format(self, record: Any) -> str:
|
123
123
|
"""Format the record into a JSON string."""
|
124
124
|
message = _make_message_dict(
|
125
|
-
record,
|
125
|
+
record,
|
126
|
+
self._fqdn,
|
127
|
+
debugging_fields=True,
|
128
|
+
extra_fields=True,
|
129
|
+
facility=None,
|
130
|
+
static_fields={},
|
126
131
|
)
|
127
132
|
return json.dumps(message)
|
c2cwsgiutils/redis_stats.py
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
import logging
|
2
2
|
import warnings
|
3
|
-
from
|
3
|
+
from collections.abc import Callable
|
4
|
+
from typing import Any
|
4
5
|
|
5
6
|
import prometheus_client
|
6
7
|
import pyramid.config
|
@@ -8,7 +9,7 @@ import pyramid.config
|
|
8
9
|
from c2cwsgiutils import config_utils, prometheus
|
9
10
|
|
10
11
|
_LOG = logging.getLogger(__name__)
|
11
|
-
_ORIG:
|
12
|
+
_ORIG: Callable[..., Any] | None = None
|
12
13
|
|
13
14
|
_PROMETHEUS_REDIS_SUMMARY = prometheus_client.Summary(
|
14
15
|
prometheus.build_metric_name("redis"),
|
@@ -24,23 +25,27 @@ def _execute_command_patch(self: Any, command: str, *args: Any, **options: Any)
|
|
24
25
|
return _ORIG(self, command, *args, **options)
|
25
26
|
|
26
27
|
|
27
|
-
def init(config:
|
28
|
+
def init(config: pyramid.config.Configurator | None = None) -> None:
|
28
29
|
"""Initialize the Redis tracking, for backward compatibility."""
|
29
30
|
warnings.warn("init function is deprecated; use includeme instead", stacklevel=2)
|
30
31
|
includeme(config)
|
31
32
|
|
32
33
|
|
33
|
-
def includeme(config:
|
34
|
+
def includeme(config: pyramid.config.Configurator | None = None) -> None:
|
34
35
|
"""Initialize the Redis tracking."""
|
35
36
|
global _ORIG # pylint: disable=global-statement
|
36
37
|
if config_utils.env_or_config(
|
37
|
-
config,
|
38
|
+
config,
|
39
|
+
"C2C_TRACK_REDIS",
|
40
|
+
"c2c.track_redis",
|
41
|
+
default=True,
|
42
|
+
type_=config_utils.config_bool,
|
38
43
|
):
|
39
44
|
try:
|
40
45
|
import redis.client # pylint: disable=import-outside-toplevel
|
41
46
|
|
42
47
|
_ORIG = redis.client.Redis.execute_command
|
43
|
-
redis.client.Redis.execute_command = _execute_command_patch # type: ignore
|
48
|
+
redis.client.Redis.execute_command = _execute_command_patch # type: ignore[method-assign,assignment]
|
44
49
|
_LOG.info("Enabled the redis tracking")
|
45
|
-
except Exception: # pragma: nocover # pylint: disable=broad-
|
50
|
+
except Exception: # pragma: nocover # pylint: disable=broad-exception-caught
|
46
51
|
_LOG.warning("Cannot enable redis tracking", exc_info=True)
|
c2cwsgiutils/redis_utils.py
CHANGED
@@ -27,7 +27,7 @@ _REDIS_DB_KEY_PROP = "c2c.redis_db"
|
|
27
27
|
|
28
28
|
_master: Optional["redis.client.Redis[str]"] = None
|
29
29
|
_slave: Optional["redis.client.Redis[str]"] = None
|
30
|
-
_sentinel:
|
30
|
+
_sentinel: redis.sentinel.Sentinel | None = None
|
31
31
|
|
32
32
|
|
33
33
|
def cleanup() -> None:
|
@@ -39,11 +39,11 @@ def cleanup() -> None:
|
|
39
39
|
|
40
40
|
|
41
41
|
def get(
|
42
|
-
settings:
|
42
|
+
settings: Mapping[str, bytes] | None = None,
|
43
43
|
) -> tuple[
|
44
44
|
Optional["redis.client.Redis[str]"],
|
45
45
|
Optional["redis.client.Redis[str]"],
|
46
|
-
|
46
|
+
redis.sentinel.Sentinel | None,
|
47
47
|
]:
|
48
48
|
"""Get the redis connection instances."""
|
49
49
|
if _master is None:
|
@@ -51,18 +51,24 @@ def get(
|
|
51
51
|
return _master, _slave, _sentinel
|
52
52
|
|
53
53
|
|
54
|
-
def _init(settings:
|
54
|
+
def _init(settings: Mapping[str, Any] | None) -> None:
|
55
55
|
global _master, _slave, _sentinel # pylint: disable=global-statement
|
56
56
|
sentinels = c2cwsgiutils.config_utils.env_or_settings(
|
57
|
-
settings,
|
57
|
+
settings,
|
58
|
+
REDIS_SENTINELS_KEY,
|
59
|
+
REDIS_SENTINELS_KEY_PROP,
|
58
60
|
)
|
59
61
|
service_name = c2cwsgiutils.config_utils.env_or_settings(
|
60
|
-
settings,
|
62
|
+
settings,
|
63
|
+
REDIS_SERVICENAME_KEY,
|
64
|
+
REDIS_SERVICENAME_KEY_PROP,
|
61
65
|
)
|
62
66
|
db = c2cwsgiutils.config_utils.env_or_settings(settings, _REDIS_DB_KEY, _REDIS_DB_KEY_PROP)
|
63
67
|
url = c2cwsgiutils.config_utils.env_or_settings(settings, REDIS_URL_KEY, REDIS_URL_KEY_PROP)
|
64
68
|
redis_options_ = c2cwsgiutils.config_utils.env_or_settings(
|
65
|
-
settings,
|
69
|
+
settings,
|
70
|
+
_REDIS_OPTIONS_KEY,
|
71
|
+
_REDIS_OPTIONS_KEY_PROP,
|
66
72
|
)
|
67
73
|
|
68
74
|
redis_options = (
|
@@ -92,14 +98,16 @@ def _init(settings: Optional[Mapping[str, Any]]) -> None:
|
|
92
98
|
_slave = _master
|
93
99
|
else:
|
94
100
|
_LOG.info(
|
95
|
-
"No Redis configuration found, use %s or %s to configure it",
|
101
|
+
"No Redis configuration found, use %s or %s to configure it",
|
102
|
+
REDIS_URL_KEY,
|
103
|
+
REDIS_SENTINELS_KEY,
|
96
104
|
)
|
97
105
|
|
98
106
|
|
99
107
|
class PubSubWorkerThread(threading.Thread):
|
100
108
|
"""A clone of redis.client.PubSubWorkerThread that doesn't die when the connections are broken."""
|
101
109
|
|
102
|
-
def __init__(self, pubsub: redis.client.PubSub, name:
|
110
|
+
def __init__(self, pubsub: redis.client.PubSub, name: str | None = None) -> None:
|
103
111
|
"""Initialize the PubSubWorkerThread."""
|
104
112
|
super().__init__(name=name, daemon=True)
|
105
113
|
self.pubsub = pubsub
|
@@ -123,7 +131,7 @@ class PubSubWorkerThread(threading.Thread):
|
|
123
131
|
_LOG.warning("Redis connection problem")
|
124
132
|
last_was_ok = False
|
125
133
|
time.sleep(0.5)
|
126
|
-
except Exception: # pylint: disable=broad-
|
134
|
+
except Exception: # pylint: disable=broad-exception-caught
|
127
135
|
_LOG.warning("Unexpected error", exc_info=True)
|
128
136
|
_LOG.info("Redis subscription worker stopped")
|
129
137
|
pubsub.close()
|