c2cwsgiutils 5.2.1__py3-none-any.whl → 5.2.1.dev197__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- c2cwsgiutils/__init__.py +12 -12
- c2cwsgiutils/acceptance/connection.py +5 -2
- c2cwsgiutils/acceptance/image.py +95 -3
- c2cwsgiutils/acceptance/package-lock.json +1933 -0
- c2cwsgiutils/acceptance/package.json +7 -0
- c2cwsgiutils/acceptance/print.py +3 -3
- c2cwsgiutils/acceptance/screenshot.js +62 -0
- c2cwsgiutils/acceptance/utils.py +14 -22
- c2cwsgiutils/auth.py +4 -4
- c2cwsgiutils/broadcast/__init__.py +15 -7
- c2cwsgiutils/broadcast/interface.py +3 -2
- c2cwsgiutils/broadcast/local.py +3 -2
- c2cwsgiutils/broadcast/redis.py +6 -5
- c2cwsgiutils/client_info.py +5 -5
- c2cwsgiutils/config_utils.py +2 -1
- c2cwsgiutils/db.py +20 -11
- c2cwsgiutils/db_maintenance_view.py +2 -1
- c2cwsgiutils/debug/_listeners.py +7 -6
- c2cwsgiutils/debug/_views.py +11 -10
- c2cwsgiutils/debug/utils.py +5 -5
- c2cwsgiutils/health_check.py +72 -73
- c2cwsgiutils/index.py +90 -105
- c2cwsgiutils/loader.py +3 -3
- c2cwsgiutils/logging_view.py +3 -2
- c2cwsgiutils/models_graph.py +4 -4
- c2cwsgiutils/prometheus.py +175 -57
- c2cwsgiutils/pyramid.py +4 -2
- c2cwsgiutils/pyramid_logging.py +2 -1
- c2cwsgiutils/redis_stats.py +13 -11
- c2cwsgiutils/redis_utils.py +11 -5
- c2cwsgiutils/request_tracking/__init__.py +36 -30
- c2cwsgiutils/scripts/genversion.py +4 -4
- c2cwsgiutils/scripts/stats_db.py +92 -60
- c2cwsgiutils/sentry.py +2 -1
- c2cwsgiutils/setup_process.py +12 -16
- c2cwsgiutils/sql_profiler/_impl.py +3 -2
- c2cwsgiutils/sqlalchemylogger/_models.py +2 -2
- c2cwsgiutils/sqlalchemylogger/handlers.py +6 -6
- c2cwsgiutils/static/favicon-16x16.png +0 -0
- c2cwsgiutils/static/favicon-32x32.png +0 -0
- c2cwsgiutils/stats_pyramid/__init__.py +7 -11
- c2cwsgiutils/stats_pyramid/_db_spy.py +14 -11
- c2cwsgiutils/stats_pyramid/_pyramid_spy.py +27 -21
- c2cwsgiutils/templates/index.html.mako +50 -0
- c2cwsgiutils/version.py +49 -16
- {c2cwsgiutils-5.2.1.dist-info → c2cwsgiutils-5.2.1.dev197.dist-info}/METADATA +168 -99
- c2cwsgiutils-5.2.1.dev197.dist-info/RECORD +67 -0
- {c2cwsgiutils-5.2.1.dist-info → c2cwsgiutils-5.2.1.dev197.dist-info}/WHEEL +1 -1
- c2cwsgiutils/acceptance/composition.py +0 -129
- c2cwsgiutils/metrics.py +0 -110
- c2cwsgiutils/scripts/check_es.py +0 -130
- c2cwsgiutils/stats.py +0 -344
- c2cwsgiutils/stats_pyramid/_views.py +0 -16
- c2cwsgiutils-5.2.1.dist-info/RECORD +0 -66
- {c2cwsgiutils-5.2.1.dist-info → c2cwsgiutils-5.2.1.dev197.dist-info}/LICENSE +0 -0
- {c2cwsgiutils-5.2.1.dist-info → c2cwsgiutils-5.2.1.dev197.dist-info}/entry_points.txt +0 -0
c2cwsgiutils/acceptance/print.py
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
import functools
|
2
2
|
import json
|
3
3
|
import logging
|
4
|
-
from typing import Any,
|
4
|
+
from typing import Any, Optional
|
5
5
|
|
6
6
|
import requests
|
7
7
|
|
@@ -32,14 +32,14 @@ class PrintConnection(connection.Connection):
|
|
32
32
|
def get_capabilities(self, app: str) -> Any:
|
33
33
|
return self.get_json(app + "/capabilities.json", cache_expected=connection.CacheExpected.YES)
|
34
34
|
|
35
|
-
def get_example_requests(self, app: str) ->
|
35
|
+
def get_example_requests(self, app: str) -> dict[str, Any]:
|
36
36
|
samples = self.get_json(app + "/exampleRequest.json", cache_expected=connection.CacheExpected.YES)
|
37
37
|
out = {}
|
38
38
|
for name, value in samples.items():
|
39
39
|
out[name] = json.loads(value)
|
40
40
|
return out
|
41
41
|
|
42
|
-
def get_pdf(self, app: str, request:
|
42
|
+
def get_pdf(self, app: str, request: dict[str, Any], timeout: int = 60) -> requests.Response:
|
43
43
|
create_report = self.post_json(app + "/report.pdf", json=request)
|
44
44
|
LOG.debug("create_report=%s", create_report)
|
45
45
|
ref = create_report["ref"]
|
@@ -0,0 +1,62 @@
|
|
1
|
+
import puppeteer from 'puppeteer';
|
2
|
+
import { program } from 'commander';
|
3
|
+
|
4
|
+
program
|
5
|
+
.option('--url <char>', 'The URL')
|
6
|
+
.option('--output <char>', 'The output filename')
|
7
|
+
.option('--width <int>', 'The page width', 800)
|
8
|
+
.option('--height <int>', 'The page height', 600)
|
9
|
+
.option('--headers <str>', 'The headers', '{}')
|
10
|
+
// see: https://pptr.dev/api/puppeteer.page.emulatemediafeatures
|
11
|
+
.option('--media <str>', 'The media feature, see Page.emulateMediaFeatures', '[]');
|
12
|
+
|
13
|
+
program.parse();
|
14
|
+
|
15
|
+
const options = program.opts();
|
16
|
+
|
17
|
+
(async () => {
|
18
|
+
const browser = await puppeteer.launch({
|
19
|
+
headless: 'new',
|
20
|
+
args: ['--no-sandbox', '--disable-web-security'],
|
21
|
+
});
|
22
|
+
const page = await browser.newPage();
|
23
|
+
page.setDefaultNavigationTimeout(60000);
|
24
|
+
await page.setExtraHTTPHeaders(JSON.parse(options.headers));
|
25
|
+
await page.emulateMediaFeatures(JSON.parse(options.media));
|
26
|
+
|
27
|
+
page.on('console', async (msg) => {
|
28
|
+
const msgArgs = msg.args();
|
29
|
+
for (let i = 0; i < msgArgs.length; ++i) {
|
30
|
+
console.log(await msgArgs[i].jsonValue());
|
31
|
+
}
|
32
|
+
});
|
33
|
+
page.on('error', (err) => {
|
34
|
+
console.log('error', err);
|
35
|
+
});
|
36
|
+
page.on('pageerror', (err) => {
|
37
|
+
console.log('pageerror', err);
|
38
|
+
});
|
39
|
+
page.on('requestfailed', (request) => {
|
40
|
+
console.log('requestfailed on URL:', request.url());
|
41
|
+
console.log(request.failure());
|
42
|
+
const response = request.response();
|
43
|
+
if (response !== null) {
|
44
|
+
console.log(response);
|
45
|
+
console.log(response.status());
|
46
|
+
console.log(response.statusText());
|
47
|
+
console.log(response.text());
|
48
|
+
}
|
49
|
+
});
|
50
|
+
|
51
|
+
await page.goto(options.url, { timeout: 60000 });
|
52
|
+
|
53
|
+
await page.setViewport({
|
54
|
+
width: parseInt(options.width),
|
55
|
+
height: parseInt(options.height),
|
56
|
+
});
|
57
|
+
await page.screenshot({
|
58
|
+
path: options.output,
|
59
|
+
clip: { x: 0, y: 0, width: parseInt(options.width), height: parseInt(options.height) },
|
60
|
+
});
|
61
|
+
await browser.close();
|
62
|
+
})();
|
c2cwsgiutils/acceptance/utils.py
CHANGED
@@ -1,33 +1,22 @@
|
|
1
1
|
import logging
|
2
|
-
import os
|
3
2
|
import time
|
4
|
-
from typing import Any, Callable
|
3
|
+
from typing import Any, Callable
|
5
4
|
|
6
|
-
import boltons.iterutils
|
7
|
-
import netifaces
|
8
5
|
import pytest
|
9
6
|
import requests
|
10
7
|
|
11
|
-
|
8
|
+
_LOG = logging.getLogger(__name__)
|
9
|
+
_DEFAULT_TIMEOUT = 60
|
12
10
|
|
13
11
|
|
14
|
-
def
|
15
|
-
"""Is in Docker mode."""
|
16
|
-
return os.environ.get("DOCKER_RUN") != "0"
|
17
|
-
|
18
|
-
|
19
|
-
DOCKER_GATEWAY = netifaces.gateways()[netifaces.AF_INET][0][0] if in_docker() else "localhost"
|
20
|
-
DEFAULT_TIMEOUT = 60
|
21
|
-
|
22
|
-
|
23
|
-
def wait_url(url: str, timeout: float = DEFAULT_TIMEOUT) -> None:
|
12
|
+
def wait_url(url: str, timeout: float = _DEFAULT_TIMEOUT) -> None:
|
24
13
|
"""Wait the the URL is available without any error."""
|
25
14
|
|
26
15
|
def what() -> bool:
|
27
|
-
|
16
|
+
_LOG.info("Trying to connect to %s... ", url)
|
28
17
|
r = requests.get(url, timeout=timeout)
|
29
18
|
if r.status_code == 200:
|
30
|
-
|
19
|
+
_LOG.info("%s service started", url)
|
31
20
|
return True
|
32
21
|
else:
|
33
22
|
return False
|
@@ -35,7 +24,7 @@ def wait_url(url: str, timeout: float = DEFAULT_TIMEOUT) -> None:
|
|
35
24
|
retry_timeout(what, timeout=timeout)
|
36
25
|
|
37
26
|
|
38
|
-
def retry_timeout(what: Callable[[], Any], timeout: float =
|
27
|
+
def retry_timeout(what: Callable[[], Any], timeout: float = _DEFAULT_TIMEOUT, interval: float = 0.5) -> Any:
|
39
28
|
"""
|
40
29
|
Retry the function until the timeout.
|
41
30
|
|
@@ -45,7 +34,8 @@ def retry_timeout(what: Callable[[], Any], timeout: float = DEFAULT_TIMEOUT, int
|
|
45
34
|
timeout: the timeout to get a success
|
46
35
|
interval: the interval between try
|
47
36
|
"""
|
48
|
-
|
37
|
+
|
38
|
+
timeout = time.perf_counter() + timeout
|
49
39
|
while True:
|
50
40
|
error = ""
|
51
41
|
try:
|
@@ -56,8 +46,8 @@ def retry_timeout(what: Callable[[], Any], timeout: float = DEFAULT_TIMEOUT, int
|
|
56
46
|
raise
|
57
47
|
except Exception as e: # pylint: disable=broad-except
|
58
48
|
error = str(e)
|
59
|
-
|
60
|
-
if time.
|
49
|
+
_LOG.info(" Failed: %s", e)
|
50
|
+
if time.perf_counter() > timeout:
|
61
51
|
assert False, "Timeout: " + error
|
62
52
|
time.sleep(interval)
|
63
53
|
|
@@ -68,10 +58,12 @@ def approx(struct: Any, **kwargs: Any) -> Any:
|
|
68
58
|
|
69
59
|
See pytest.approx
|
70
60
|
"""
|
61
|
+
import boltons.iterutils
|
62
|
+
|
71
63
|
if isinstance(struct, float):
|
72
64
|
return pytest.approx(struct, **kwargs)
|
73
65
|
|
74
|
-
def visit(_path:
|
66
|
+
def visit(_path: list[str], key: Any, value: Any) -> tuple[Any, Any]:
|
75
67
|
if isinstance(value, float):
|
76
68
|
value = pytest.approx(value, **kwargs)
|
77
69
|
return key, value
|
c2cwsgiutils/auth.py
CHANGED
@@ -1,7 +1,8 @@
|
|
1
1
|
import hashlib
|
2
2
|
import logging
|
3
|
+
from collections.abc import Mapping
|
3
4
|
from enum import Enum
|
4
|
-
from typing import Any,
|
5
|
+
from typing import Any, Optional, TypedDict, cast
|
5
6
|
|
6
7
|
import jwt
|
7
8
|
import pyramid.request
|
@@ -98,8 +99,7 @@ def _is_auth_secret(request: pyramid.request.Request) -> bool:
|
|
98
99
|
return False
|
99
100
|
|
100
101
|
|
101
|
-
def _is_auth_user_github(request: pyramid.request.Request) ->
|
102
|
-
|
102
|
+
def _is_auth_user_github(request: pyramid.request.Request) -> tuple[bool, UserDetails]:
|
103
103
|
settings = request.registry.settings
|
104
104
|
cookie = request.cookies.get(
|
105
105
|
env_or_settings(
|
@@ -129,7 +129,7 @@ def _is_auth_user_github(request: pyramid.request.Request) -> Tuple[bool, UserDe
|
|
129
129
|
return False, {}
|
130
130
|
|
131
131
|
|
132
|
-
def is_auth_user(request: pyramid.request.Request) ->
|
132
|
+
def is_auth_user(request: pyramid.request.Request) -> tuple[bool, UserDetails]:
|
133
133
|
"""
|
134
134
|
Check if the client is authenticated.
|
135
135
|
|
@@ -2,7 +2,7 @@
|
|
2
2
|
import functools
|
3
3
|
import logging
|
4
4
|
import warnings
|
5
|
-
from typing import Any, Callable,
|
5
|
+
from typing import Any, Callable, Optional, TypeVar
|
6
6
|
|
7
7
|
import pyramid.config
|
8
8
|
|
@@ -36,11 +36,12 @@ def includeme(config: Optional[pyramid.config.Configurator] = None) -> None:
|
|
36
36
|
if _broadcaster is None:
|
37
37
|
if master is not None and slave is not None:
|
38
38
|
_broadcaster = redis.RedisBroadcaster(broadcast_prefix, master, slave)
|
39
|
+
LOG.info("Broadcast service setup using Redis implementation")
|
39
40
|
else:
|
40
41
|
_broadcaster = local.LocalBroadcaster()
|
41
42
|
LOG.info("Broadcast service setup using local implementation")
|
42
43
|
elif isinstance(_broadcaster, local.LocalBroadcaster) and master is not None and slave is not None:
|
43
|
-
LOG.info("Switching from a local broadcaster to a
|
44
|
+
LOG.info("Switching from a local broadcaster to a Redis broadcaster")
|
44
45
|
prev_broadcaster = _broadcaster
|
45
46
|
_broadcaster = redis.RedisBroadcaster(broadcast_prefix, master, slave)
|
46
47
|
_broadcaster.copy_local_subscriptions(prev_broadcaster)
|
@@ -55,6 +56,13 @@ def _get(need_init: bool = False) -> interface.BaseBroadcaster:
|
|
55
56
|
return _broadcaster
|
56
57
|
|
57
58
|
|
59
|
+
def cleanup() -> None:
|
60
|
+
"""Cleanup the broadcaster to force to reinitialize it."""
|
61
|
+
|
62
|
+
global _broadcaster
|
63
|
+
_broadcaster = None
|
64
|
+
|
65
|
+
|
58
66
|
def subscribe(channel: str, callback: Callable[..., Any]) -> None:
|
59
67
|
"""
|
60
68
|
Subscribe to a broadcast channel with the given callback.
|
@@ -73,8 +81,8 @@ def unsubscribe(channel: str) -> None:
|
|
73
81
|
|
74
82
|
|
75
83
|
def broadcast(
|
76
|
-
channel: str, params: Optional[
|
77
|
-
) -> Optional[
|
84
|
+
channel: str, params: Optional[dict[str, Any]] = None, expect_answers: bool = False, timeout: float = 10
|
85
|
+
) -> Optional[list[Any]]:
|
78
86
|
"""
|
79
87
|
Broadcast a message to the given channel.
|
80
88
|
|
@@ -92,16 +100,16 @@ _DECORATOR_RETURN = TypeVar("_DECORATOR_RETURN")
|
|
92
100
|
|
93
101
|
def decorator(
|
94
102
|
channel: Optional[str] = None, expect_answers: bool = False, timeout: float = 10
|
95
|
-
) -> Callable[[Callable[..., _DECORATOR_RETURN]], Callable[..., Optional[
|
103
|
+
) -> Callable[[Callable[..., _DECORATOR_RETURN]], Callable[..., Optional[list[_DECORATOR_RETURN]]]]:
|
96
104
|
"""
|
97
105
|
Decorate function will be called through the broadcast functionality.
|
98
106
|
|
99
107
|
If expect_answers is set to True, the returned value will be a list of all the answers.
|
100
108
|
"""
|
101
109
|
|
102
|
-
def impl(func: Callable[..., _DECORATOR_RETURN]) -> Callable[..., Optional[
|
110
|
+
def impl(func: Callable[..., _DECORATOR_RETURN]) -> Callable[..., Optional[list[_DECORATOR_RETURN]]]:
|
103
111
|
@functools.wraps(func)
|
104
|
-
def wrapper(**kwargs: Any) -> Optional[
|
112
|
+
def wrapper(**kwargs: Any) -> Optional[list[_DECORATOR_RETURN]]:
|
105
113
|
return broadcast(_channel, params=kwargs, expect_answers=expect_answers, timeout=timeout)
|
106
114
|
|
107
115
|
if channel is None:
|
@@ -1,5 +1,6 @@
|
|
1
1
|
from abc import abstractmethod
|
2
|
-
from
|
2
|
+
from collections.abc import Mapping
|
3
|
+
from typing import Any, Callable, Optional
|
3
4
|
|
4
5
|
|
5
6
|
class BaseBroadcaster:
|
@@ -16,5 +17,5 @@ class BaseBroadcaster:
|
|
16
17
|
@abstractmethod
|
17
18
|
def broadcast(
|
18
19
|
self, channel: str, params: Mapping[str, Any], expect_answers: bool, timeout: float
|
19
|
-
) -> Optional[
|
20
|
+
) -> Optional[list[Any]]:
|
20
21
|
pass # pragma: no cover
|
c2cwsgiutils/broadcast/local.py
CHANGED
@@ -1,4 +1,5 @@
|
|
1
|
-
from
|
1
|
+
from collections.abc import Mapping, MutableMapping
|
2
|
+
from typing import Any, Callable, Optional
|
2
3
|
|
3
4
|
# noinspection PyProtectedMember
|
4
5
|
from c2cwsgiutils.broadcast import interface, utils
|
@@ -18,7 +19,7 @@ class LocalBroadcaster(interface.BaseBroadcaster):
|
|
18
19
|
|
19
20
|
def broadcast(
|
20
21
|
self, channel: str, params: Mapping[str, Any], expect_answers: bool, timeout: float
|
21
|
-
) -> Optional[
|
22
|
+
) -> Optional[list[Any]]:
|
22
23
|
subscriber = self._subscribers.get(channel, None)
|
23
24
|
answers = [utils.add_host_info(subscriber(**params))] if subscriber is not None else []
|
24
25
|
return answers if expect_answers else None
|
c2cwsgiutils/broadcast/redis.py
CHANGED
@@ -4,7 +4,8 @@ import random
|
|
4
4
|
import string
|
5
5
|
import threading
|
6
6
|
import time
|
7
|
-
from
|
7
|
+
from collections.abc import Mapping
|
8
|
+
from typing import Any, Callable, Optional
|
8
9
|
|
9
10
|
import redis
|
10
11
|
|
@@ -63,7 +64,7 @@ class RedisBroadcaster(interface.BaseBroadcaster):
|
|
63
64
|
|
64
65
|
def broadcast(
|
65
66
|
self, channel: str, params: Mapping[str, Any], expect_answers: bool, timeout: float
|
66
|
-
) -> Optional[
|
67
|
+
) -> Optional[list[Any]]:
|
67
68
|
if expect_answers:
|
68
69
|
return self._broadcast_with_answer(channel, params, timeout)
|
69
70
|
else:
|
@@ -72,7 +73,7 @@ class RedisBroadcaster(interface.BaseBroadcaster):
|
|
72
73
|
|
73
74
|
def _broadcast_with_answer(
|
74
75
|
self, channel: str, params: Optional[Mapping[str, Any]], timeout: float
|
75
|
-
) ->
|
76
|
+
) -> list[Any]:
|
76
77
|
cond = threading.Condition()
|
77
78
|
answers = []
|
78
79
|
assert self._thread.is_alive()
|
@@ -93,10 +94,10 @@ class RedisBroadcaster(interface.BaseBroadcaster):
|
|
93
94
|
try:
|
94
95
|
nb_received = self._broadcast(channel, message)
|
95
96
|
|
96
|
-
timeout_time = time.
|
97
|
+
timeout_time = time.perf_counter() + timeout
|
97
98
|
with cond:
|
98
99
|
while len(answers) < nb_received:
|
99
|
-
to_wait = timeout_time - time.
|
100
|
+
to_wait = timeout_time - time.perf_counter()
|
100
101
|
if to_wait <= 0.0: # pragma: no cover
|
101
102
|
LOG.warning(
|
102
103
|
"timeout waiting for %d/%d answers on %s",
|
c2cwsgiutils/client_info.py
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
import re
|
2
|
-
from typing import Any, Callable
|
2
|
+
from typing import Any, Callable
|
3
3
|
|
4
4
|
SEP_RE = re.compile(r", *")
|
5
5
|
|
@@ -12,10 +12,10 @@ class Filter:
|
|
12
12
|
Concerned headers: Forwarded and the X_Forwarded_* Headers.
|
13
13
|
"""
|
14
14
|
|
15
|
-
def __init__(self, application: Callable[[
|
15
|
+
def __init__(self, application: Callable[[dict[str, str], Any], Any]):
|
16
16
|
self._application = application
|
17
17
|
|
18
|
-
def __call__(self, environ:
|
18
|
+
def __call__(self, environ: dict[str, str], start_response: Any) -> Any:
|
19
19
|
# https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Forwarded
|
20
20
|
if "HTTP_FORWARDED" in environ:
|
21
21
|
_handle_forwarded(environ)
|
@@ -25,7 +25,7 @@ class Filter:
|
|
25
25
|
return self._application(environ, start_response)
|
26
26
|
|
27
27
|
|
28
|
-
def _handle_others(environ:
|
28
|
+
def _handle_others(environ: dict[str, str]) -> None:
|
29
29
|
# The rest is taken from paste.deploy.config.PrefixMiddleware
|
30
30
|
if "HTTP_X_FORWARDED_SERVER" in environ:
|
31
31
|
environ["HTTP_ORIGINAL_X_FORWARDED_SERVER"] = environ["HTTP_X_FORWARDED_SERVER"]
|
@@ -45,7 +45,7 @@ def _handle_others(environ: Dict[str, str]) -> None:
|
|
45
45
|
environ["wsgi.url_scheme"] = environ.pop("HTTP_X_FORWARDED_PROTO")
|
46
46
|
|
47
47
|
|
48
|
-
def _handle_forwarded(environ:
|
48
|
+
def _handle_forwarded(environ: dict[str, str]) -> None:
|
49
49
|
environ["HTTP_ORIGINAL_FORWARDED"] = environ["HTTP_FORWARDED"]
|
50
50
|
for header in (
|
51
51
|
"X_FORWARDED_SERVER",
|
c2cwsgiutils/config_utils.py
CHANGED
c2cwsgiutils/db.py
CHANGED
@@ -2,7 +2,9 @@
|
|
2
2
|
import logging
|
3
3
|
import re
|
4
4
|
import warnings
|
5
|
-
from
|
5
|
+
from collections.abc import Iterable
|
6
|
+
from re import Pattern
|
7
|
+
from typing import TYPE_CHECKING, Any, Callable, Optional, Union, cast
|
6
8
|
|
7
9
|
import pyramid.config
|
8
10
|
import pyramid.config.settings
|
@@ -12,9 +14,16 @@ import sqlalchemy.orm
|
|
12
14
|
import transaction
|
13
15
|
import zope.sqlalchemy
|
14
16
|
from sqlalchemy import engine_from_config
|
15
|
-
from sqlalchemy.orm import sessionmaker
|
16
17
|
from zope.sqlalchemy import register
|
17
18
|
|
19
|
+
if TYPE_CHECKING:
|
20
|
+
scoped_session = sqlalchemy.orm.scoped_session[sqlalchemy.orm.Session]
|
21
|
+
sessionmaker = sqlalchemy.orm.sessionmaker[sqlalchemy.orm.Session]
|
22
|
+
else:
|
23
|
+
scoped_session = sqlalchemy.orm.scoped_session
|
24
|
+
sessionmaker = sqlalchemy.orm.sessionmaker
|
25
|
+
|
26
|
+
|
18
27
|
LOG = logging.getLogger(__name__)
|
19
28
|
RE_COMPILE: Callable[[str], Pattern[str]] = re.compile
|
20
29
|
|
@@ -34,8 +43,8 @@ def setup_session(
|
|
34
43
|
slave_prefix: Optional[str] = None,
|
35
44
|
force_master: Optional[Iterable[str]] = None,
|
36
45
|
force_slave: Optional[Iterable[str]] = None,
|
37
|
-
) ->
|
38
|
-
Union[sqlalchemy.orm.Session,
|
46
|
+
) -> tuple[
|
47
|
+
Union[sqlalchemy.orm.Session, scoped_session],
|
39
48
|
sqlalchemy.engine.Engine,
|
40
49
|
sqlalchemy.engine.Engine,
|
41
50
|
]:
|
@@ -95,7 +104,7 @@ def create_session(
|
|
95
104
|
force_master: Optional[Iterable[str]] = None,
|
96
105
|
force_slave: Optional[Iterable[str]] = None,
|
97
106
|
**engine_config: Any,
|
98
|
-
) -> Union[sqlalchemy.orm.Session,
|
107
|
+
) -> Union[sqlalchemy.orm.Session, scoped_session]:
|
99
108
|
"""
|
100
109
|
Create a SQLAlchemy session.
|
101
110
|
|
@@ -147,7 +156,7 @@ def create_session(
|
|
147
156
|
def _add_tween(
|
148
157
|
config: pyramid.config.Configurator,
|
149
158
|
name: str,
|
150
|
-
db_session:
|
159
|
+
db_session: scoped_session,
|
151
160
|
force_master: Optional[Iterable[str]],
|
152
161
|
force_slave: Optional[Iterable[str]],
|
153
162
|
) -> None:
|
@@ -201,7 +210,7 @@ def _add_tween(
|
|
201
210
|
config.add_tween("c2cwsgiutils.db.tweens." + name, over="pyramid_tm.tm_tween_factory")
|
202
211
|
|
203
212
|
|
204
|
-
class SessionFactory(sessionmaker
|
213
|
+
class SessionFactory(sessionmaker):
|
205
214
|
"""The custom session factory that manage the read only and read write sessions."""
|
206
215
|
|
207
216
|
def __init__(
|
@@ -226,7 +235,7 @@ class SessionFactory(sessionmaker[sqlalchemy.orm.Session]): # pylint: disable=u
|
|
226
235
|
|
227
236
|
def __call__( # type: ignore
|
228
237
|
self, request: Optional[pyramid.request.Request], readwrite: Optional[bool] = None, **local_kw: Any
|
229
|
-
) ->
|
238
|
+
) -> scoped_session:
|
230
239
|
if readwrite is not None:
|
231
240
|
if readwrite and not force_readonly:
|
232
241
|
LOG.debug("Using %s database", self.rw_engine.c2c_name) # type: ignore
|
@@ -262,7 +271,7 @@ def get_engine(
|
|
262
271
|
|
263
272
|
def get_session_factory(
|
264
273
|
engine: sqlalchemy.engine.Engine,
|
265
|
-
) -> sessionmaker
|
274
|
+
) -> sessionmaker:
|
266
275
|
"""Get the session factory from the engine."""
|
267
276
|
factory = sessionmaker()
|
268
277
|
factory.configure(bind=engine)
|
@@ -270,7 +279,7 @@ def get_session_factory(
|
|
270
279
|
|
271
280
|
|
272
281
|
def get_tm_session(
|
273
|
-
session_factory: sessionmaker
|
282
|
+
session_factory: sessionmaker,
|
274
283
|
transaction_manager: transaction.TransactionManager,
|
275
284
|
) -> sqlalchemy.orm.Session:
|
276
285
|
"""
|
@@ -338,7 +347,7 @@ def get_tm_session_pyramid(
|
|
338
347
|
session_factory: SessionFactory,
|
339
348
|
transaction_manager: transaction.TransactionManager,
|
340
349
|
request: pyramid.request.Request,
|
341
|
-
) ->
|
350
|
+
) -> scoped_session:
|
342
351
|
"""
|
343
352
|
Get a ``sqlalchemy.orm.Session`` instance backed by a transaction.
|
344
353
|
|
c2cwsgiutils/debug/_listeners.py
CHANGED
@@ -3,7 +3,8 @@ import sys
|
|
3
3
|
import threading
|
4
4
|
import time
|
5
5
|
import traceback
|
6
|
-
from
|
6
|
+
from collections.abc import Mapping
|
7
|
+
from typing import Any, Optional, cast
|
7
8
|
|
8
9
|
import objgraph
|
9
10
|
|
@@ -13,7 +14,7 @@ from c2cwsgiutils.debug.utils import get_size
|
|
13
14
|
FILES_FIELDS = {"__name__", "__doc__", "__package__", "__loader__", "__spec__", "__file__"}
|
14
15
|
|
15
16
|
|
16
|
-
def _dump_stacks_impl() ->
|
17
|
+
def _dump_stacks_impl() -> dict[str, Any]:
|
17
18
|
id2name = {th.ident: th.name for th in threading.enumerate()}
|
18
19
|
threads = {}
|
19
20
|
for thread_id, stack in sys._current_frames().items(): # pylint: disable=W0212
|
@@ -47,10 +48,10 @@ def _dump_memory_impl(
|
|
47
48
|
|
48
49
|
if analyze_type:
|
49
50
|
# timeout after one minute, must be set to a bit less that the timeout of the broadcast in _views.py
|
50
|
-
timeout = time.
|
51
|
+
timeout = time.perf_counter() + 60
|
51
52
|
|
52
|
-
mod_counts:
|
53
|
-
biggest_objects:
|
53
|
+
mod_counts: dict[str, int] = {}
|
54
|
+
biggest_objects: list[tuple[float, Any]] = []
|
54
55
|
result[analyze_type] = {}
|
55
56
|
for obj in objgraph.by_type(analyze_type):
|
56
57
|
if analyze_type == "builtins.function":
|
@@ -80,7 +81,7 @@ def _dump_memory_impl(
|
|
80
81
|
biggest_objects.sort(key=lambda x: x[0])
|
81
82
|
if len(biggest_objects) > limit:
|
82
83
|
biggest_objects = biggest_objects[-limit:]
|
83
|
-
if time.
|
84
|
+
if time.perf_counter() > timeout:
|
84
85
|
result[analyze_type]["timeout"] = True
|
85
86
|
break
|
86
87
|
if analyze_type == "builtins.function":
|
c2cwsgiutils/debug/_views.py
CHANGED
@@ -2,9 +2,10 @@ import gc
|
|
2
2
|
import logging
|
3
3
|
import re
|
4
4
|
import time
|
5
|
+
from collections.abc import Mapping
|
5
6
|
from datetime import datetime
|
6
7
|
from io import StringIO
|
7
|
-
from typing import Any, Callable,
|
8
|
+
from typing import Any, Callable, cast
|
8
9
|
|
9
10
|
import objgraph
|
10
11
|
import pyramid.config
|
@@ -19,9 +20,9 @@ LOG = logging.getLogger(__name__)
|
|
19
20
|
SPACE_RE = re.compile(r" +")
|
20
21
|
|
21
22
|
|
22
|
-
def _beautify_stacks(source:
|
23
|
+
def _beautify_stacks(source: list[Mapping[str, Any]]) -> list[Mapping[str, Any]]:
|
23
24
|
"""Group the identical stacks together along with a list of threads sporting them."""
|
24
|
-
results:
|
25
|
+
results: list[Mapping[str, Any]] = []
|
25
26
|
for host_stacks in source:
|
26
27
|
host_id = f"{host_stacks['hostname']}/{host_stacks['pid']:d}"
|
27
28
|
for thread, frames in host_stacks["threads"].items():
|
@@ -35,14 +36,14 @@ def _beautify_stacks(source: List[Mapping[str, Any]]) -> List[Mapping[str, Any]]
|
|
35
36
|
return results
|
36
37
|
|
37
38
|
|
38
|
-
def _dump_stacks(request: pyramid.request.Request) ->
|
39
|
+
def _dump_stacks(request: pyramid.request.Request) -> list[Mapping[str, Any]]:
|
39
40
|
auth.auth_view(request)
|
40
41
|
result = broadcast.broadcast("c2c_dump_stacks", expect_answers=True)
|
41
42
|
assert result is not None
|
42
43
|
return _beautify_stacks(result)
|
43
44
|
|
44
45
|
|
45
|
-
def _dump_memory(request: pyramid.request.Request) ->
|
46
|
+
def _dump_memory(request: pyramid.request.Request) -> list[Mapping[str, Any]]:
|
46
47
|
auth.auth_view(request)
|
47
48
|
limit = int(request.params.get("limit", "30"))
|
48
49
|
analyze_type = request.params.get("analyze_type")
|
@@ -57,7 +58,7 @@ def _dump_memory(request: pyramid.request.Request) -> List[Mapping[str, Any]]:
|
|
57
58
|
return result
|
58
59
|
|
59
60
|
|
60
|
-
def _dump_memory_diff(request: pyramid.request.Request) ->
|
61
|
+
def _dump_memory_diff(request: pyramid.request.Request) -> list[Any]:
|
61
62
|
auth.auth_view(request)
|
62
63
|
limit = int(request.params.get("limit", "30"))
|
63
64
|
if "path" in request.matchdict:
|
@@ -81,7 +82,7 @@ def _dump_memory_diff(request: pyramid.request.Request) -> List[Any]:
|
|
81
82
|
|
82
83
|
LOG.debug("checking memory growth for %s", path)
|
83
84
|
|
84
|
-
peak_stats:
|
85
|
+
peak_stats: dict[Any, Any] = {}
|
85
86
|
for i in range(3):
|
86
87
|
gc.collect(i)
|
87
88
|
|
@@ -158,7 +159,7 @@ def _add_view(
|
|
158
159
|
config.add_view(view, route_name="c2c_debug_" + name, renderer="fast_json", http_cache=0)
|
159
160
|
|
160
161
|
|
161
|
-
def _dump_memory_maps(request: pyramid.request.Request) ->
|
162
|
+
def _dump_memory_maps(request: pyramid.request.Request) -> list[dict[str, Any]]:
|
162
163
|
auth.auth_view(request)
|
163
164
|
return sorted(dump_memory_maps(), key=lambda i: cast(int, -i.get("pss_kb", 0)))
|
164
165
|
|
@@ -168,13 +169,13 @@ def _show_refs(request: pyramid.request.Request) -> pyramid.response.Response:
|
|
168
169
|
for generation in range(3):
|
169
170
|
gc.collect(generation)
|
170
171
|
|
171
|
-
objs:
|
172
|
+
objs: list[Any] = []
|
172
173
|
if "analyze_type" in request.params:
|
173
174
|
objs = objgraph.by_type(request.params["analyze_type"])
|
174
175
|
elif "analyze_id" in request.params:
|
175
176
|
objs = [objgraph.by(int(request.params["analyze_id"]))]
|
176
177
|
|
177
|
-
args:
|
178
|
+
args: dict[str, Any] = {
|
178
179
|
"refcounts": True,
|
179
180
|
}
|
180
181
|
if request.params.get("max_depth", "") != "":
|
c2cwsgiutils/debug/utils.py
CHANGED
@@ -5,7 +5,7 @@ import re
|
|
5
5
|
import sys
|
6
6
|
from collections import defaultdict
|
7
7
|
from types import FunctionType, ModuleType
|
8
|
-
from typing import Any
|
8
|
+
from typing import Any
|
9
9
|
|
10
10
|
# 7ff7d33bd000-7ff7d33be000 r--p 00000000 00:65 49 /usr/lib/toto.so
|
11
11
|
SMAPS_LOCATION_RE = re.compile(r"^[0-9a-f]+-[0-9a-f]+ +.... +[0-9a-f]+ +[^ ]+ +\d+ +(.*)$")
|
@@ -21,7 +21,7 @@ def get_size(obj: Any) -> int:
|
|
21
21
|
"""Get the sum size of object & members."""
|
22
22
|
if isinstance(obj, BLACKLIST):
|
23
23
|
return 0
|
24
|
-
seen_ids:
|
24
|
+
seen_ids: set[int] = set()
|
25
25
|
size = 0
|
26
26
|
objects = [obj]
|
27
27
|
while objects:
|
@@ -35,14 +35,14 @@ def get_size(obj: Any) -> int:
|
|
35
35
|
return size
|
36
36
|
|
37
37
|
|
38
|
-
def dump_memory_maps(pid: str = "self") ->
|
38
|
+
def dump_memory_maps(pid: str = "self") -> list[dict[str, Any]]:
|
39
39
|
"""Get the Linux memory maps."""
|
40
40
|
filename = os.path.join("/proc", pid, "smaps")
|
41
41
|
if not os.path.exists(filename):
|
42
42
|
return []
|
43
43
|
with open(filename, encoding="utf-8") as input_:
|
44
|
-
cur_dict:
|
45
|
-
sizes:
|
44
|
+
cur_dict: dict[str, int] = defaultdict(int)
|
45
|
+
sizes: dict[str, Any] = {}
|
46
46
|
for line in input_:
|
47
47
|
line = line.rstrip("\n")
|
48
48
|
matcher = SMAPS_LOCATION_RE.match(line)
|