c2cwsgiutils 6.2.0.dev74__py3-none-any.whl → 6.2.0.dev76__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- c2cwsgiutils/__init__.py +4 -4
- c2cwsgiutils/acceptance/__init__.py +4 -1
- c2cwsgiutils/acceptance/connection.py +45 -32
- c2cwsgiutils/acceptance/image.py +45 -39
- c2cwsgiutils/acceptance/print.py +2 -2
- c2cwsgiutils/acceptance/utils.py +4 -4
- c2cwsgiutils/auth.py +20 -13
- c2cwsgiutils/broadcast/__init__.py +25 -16
- c2cwsgiutils/broadcast/interface.py +8 -4
- c2cwsgiutils/broadcast/local.py +9 -4
- c2cwsgiutils/broadcast/redis.py +19 -12
- c2cwsgiutils/client_info.py +3 -2
- c2cwsgiutils/config_utils.py +14 -10
- c2cwsgiutils/coverage_setup.py +5 -5
- c2cwsgiutils/db.py +54 -47
- c2cwsgiutils/db_maintenance_view.py +7 -8
- c2cwsgiutils/debug/__init__.py +1 -2
- c2cwsgiutils/debug/_listeners.py +6 -4
- c2cwsgiutils/debug/_views.py +15 -10
- c2cwsgiutils/debug/utils.py +5 -5
- c2cwsgiutils/errors.py +15 -10
- c2cwsgiutils/health_check.py +79 -60
- c2cwsgiutils/index.py +35 -37
- c2cwsgiutils/loader.py +5 -4
- c2cwsgiutils/logging_view.py +13 -6
- c2cwsgiutils/models_graph.py +5 -5
- c2cwsgiutils/pretty_json.py +3 -2
- c2cwsgiutils/profiler.py +1 -2
- c2cwsgiutils/prometheus.py +6 -6
- c2cwsgiutils/pyramid.py +1 -1
- c2cwsgiutils/pyramid_logging.py +9 -4
- c2cwsgiutils/redis_stats.py +12 -7
- c2cwsgiutils/redis_utils.py +18 -10
- c2cwsgiutils/request_tracking/__init__.py +20 -12
- c2cwsgiutils/request_tracking/_sql.py +2 -1
- c2cwsgiutils/scripts/genversion.py +10 -9
- c2cwsgiutils/scripts/stats_db.py +29 -13
- c2cwsgiutils/sentry.py +44 -20
- c2cwsgiutils/setup_process.py +7 -4
- c2cwsgiutils/sql_profiler/_impl.py +12 -11
- c2cwsgiutils/sqlalchemylogger/_models.py +3 -3
- c2cwsgiutils/sqlalchemylogger/examples/__init__.py +0 -0
- c2cwsgiutils/sqlalchemylogger/handlers.py +8 -9
- c2cwsgiutils/stats_pyramid/_db_spy.py +12 -3
- c2cwsgiutils/stats_pyramid/_pyramid_spy.py +10 -9
- c2cwsgiutils/version.py +10 -7
- {c2cwsgiutils-6.2.0.dev74.dist-info → c2cwsgiutils-6.2.0.dev76.dist-info}/METADATA +1 -1
- c2cwsgiutils-6.2.0.dev76.dist-info/RECORD +68 -0
- c2cwsgiutils-6.2.0.dev74.dist-info/RECORD +0 -67
- {c2cwsgiutils-6.2.0.dev74.dist-info → c2cwsgiutils-6.2.0.dev76.dist-info}/LICENSE +0 -0
- {c2cwsgiutils-6.2.0.dev74.dist-info → c2cwsgiutils-6.2.0.dev76.dist-info}/WHEEL +0 -0
- {c2cwsgiutils-6.2.0.dev74.dist-info → c2cwsgiutils-6.2.0.dev76.dist-info}/entry_points.txt +0 -0
@@ -3,7 +3,8 @@
|
|
3
3
|
import functools
|
4
4
|
import logging
|
5
5
|
import warnings
|
6
|
-
from
|
6
|
+
from collections.abc import Callable
|
7
|
+
from typing import Any, TypeVar
|
7
8
|
|
8
9
|
import pyramid.config
|
9
10
|
|
@@ -14,16 +15,16 @@ _LOG = logging.getLogger(__name__)
|
|
14
15
|
_BROADCAST_ENV_KEY = "C2C_BROADCAST_PREFIX"
|
15
16
|
_BROADCAST_CONFIG_KEY = "c2c.broadcast_prefix"
|
16
17
|
|
17
|
-
_broadcaster:
|
18
|
+
_broadcaster: interface.BaseBroadcaster | None = None
|
18
19
|
|
19
20
|
|
20
|
-
def init(config:
|
21
|
+
def init(config: pyramid.config.Configurator | None = None) -> None:
|
21
22
|
"""Initialize the broadcaster with Redis, if configured, for backward compatibility."""
|
22
23
|
warnings.warn("init function is deprecated; use includeme instead", stacklevel=2)
|
23
24
|
includeme(config)
|
24
25
|
|
25
26
|
|
26
|
-
def includeme(config:
|
27
|
+
def includeme(config: pyramid.config.Configurator | None = None) -> None:
|
27
28
|
"""
|
28
29
|
Initialize the broadcaster with Redis, if configured.
|
29
30
|
|
@@ -31,7 +32,10 @@ def includeme(config: Optional[pyramid.config.Configurator] = None) -> None:
|
|
31
32
|
"""
|
32
33
|
global _broadcaster # pylint: disable=global-statement
|
33
34
|
broadcast_prefix = config_utils.env_or_config(
|
34
|
-
config,
|
35
|
+
config,
|
36
|
+
_BROADCAST_ENV_KEY,
|
37
|
+
_BROADCAST_CONFIG_KEY,
|
38
|
+
"broadcast_api_",
|
35
39
|
)
|
36
40
|
master, slave, _ = redis_utils.get(config.get_settings() if config else None)
|
37
41
|
if _broadcaster is None:
|
@@ -81,15 +85,21 @@ def unsubscribe(channel: str) -> None:
|
|
81
85
|
|
82
86
|
|
83
87
|
def broadcast(
|
84
|
-
channel: str,
|
85
|
-
|
88
|
+
channel: str,
|
89
|
+
params: dict[str, Any] | None = None,
|
90
|
+
expect_answers: bool = False,
|
91
|
+
timeout: float = 10,
|
92
|
+
) -> list[Any] | None:
|
86
93
|
"""
|
87
94
|
Broadcast a message to the given channel.
|
88
95
|
|
89
96
|
If answers are expected, it will wait up to "timeout" seconds to get all the answers.
|
90
97
|
"""
|
91
98
|
return _get(need_init=True).broadcast(
|
92
|
-
channel,
|
99
|
+
channel,
|
100
|
+
params if params is not None else {},
|
101
|
+
expect_answers,
|
102
|
+
timeout,
|
93
103
|
)
|
94
104
|
|
95
105
|
|
@@ -99,23 +109,22 @@ _DecoratorReturn = TypeVar("_DecoratorReturn")
|
|
99
109
|
|
100
110
|
|
101
111
|
def decorator(
|
102
|
-
channel:
|
103
|
-
|
112
|
+
channel: str | None = None,
|
113
|
+
expect_answers: bool = False,
|
114
|
+
timeout: float = 10,
|
115
|
+
) -> Callable[[Callable[..., _DecoratorReturn]], Callable[..., list[_DecoratorReturn] | None]]:
|
104
116
|
"""
|
105
117
|
Decorate function will be called through the broadcast functionality.
|
106
118
|
|
107
119
|
If expect_answers is set to True, the returned value will be a list of all the answers.
|
108
120
|
"""
|
109
121
|
|
110
|
-
def impl(func: Callable[..., _DecoratorReturn]) -> Callable[...,
|
122
|
+
def impl(func: Callable[..., _DecoratorReturn]) -> Callable[..., list[_DecoratorReturn] | None]:
|
111
123
|
@functools.wraps(func)
|
112
|
-
def wrapper(**kwargs: Any) ->
|
124
|
+
def wrapper(**kwargs: Any) -> list[_DecoratorReturn] | None:
|
113
125
|
return broadcast(_channel, params=kwargs, expect_answers=expect_answers, timeout=timeout)
|
114
126
|
|
115
|
-
if channel is None
|
116
|
-
_channel = f"c2c_decorated_{func.__module__}.{func.__name__}"
|
117
|
-
else:
|
118
|
-
_channel = channel
|
127
|
+
_channel = f"c2c_decorated_{func.__module__}.{func.__name__}" if channel is None else channel
|
119
128
|
subscribe(_channel, func)
|
120
129
|
|
121
130
|
return wrapper
|
@@ -1,6 +1,6 @@
|
|
1
1
|
from abc import abstractmethod
|
2
|
-
from collections.abc import Mapping
|
3
|
-
from typing import Any
|
2
|
+
from collections.abc import Callable, Mapping
|
3
|
+
from typing import Any
|
4
4
|
|
5
5
|
|
6
6
|
class BaseBroadcaster:
|
@@ -16,6 +16,10 @@ class BaseBroadcaster:
|
|
16
16
|
|
17
17
|
@abstractmethod
|
18
18
|
def broadcast(
|
19
|
-
self,
|
20
|
-
|
19
|
+
self,
|
20
|
+
channel: str,
|
21
|
+
params: Mapping[str, Any],
|
22
|
+
expect_answers: bool,
|
23
|
+
timeout: float,
|
24
|
+
) -> list[Any] | None:
|
21
25
|
"""Broadcast a message to a channel."""
|
c2cwsgiutils/broadcast/local.py
CHANGED
@@ -1,5 +1,5 @@
|
|
1
|
-
from collections.abc import Mapping, MutableMapping
|
2
|
-
from typing import Any
|
1
|
+
from collections.abc import Callable, Mapping, MutableMapping
|
2
|
+
from typing import Any
|
3
3
|
|
4
4
|
# noinspection PyProtectedMember
|
5
5
|
from c2cwsgiutils.broadcast import interface, utils
|
@@ -21,9 +21,14 @@ class LocalBroadcaster(interface.BaseBroadcaster):
|
|
21
21
|
del self._subscribers[channel]
|
22
22
|
|
23
23
|
def broadcast(
|
24
|
-
self,
|
25
|
-
|
24
|
+
self,
|
25
|
+
channel: str,
|
26
|
+
params: Mapping[str, Any],
|
27
|
+
expect_answers: bool,
|
28
|
+
timeout: float,
|
29
|
+
) -> list[Any] | None:
|
26
30
|
"""Broadcast a message to all the listeners."""
|
31
|
+
del timeout # Not used
|
27
32
|
subscriber = self._subscribers.get(channel, None)
|
28
33
|
answers = [utils.add_host_info(subscriber(**params))] if subscriber is not None else []
|
29
34
|
return answers if expect_answers else None
|
c2cwsgiutils/broadcast/redis.py
CHANGED
@@ -4,13 +4,14 @@ import random
|
|
4
4
|
import string
|
5
5
|
import threading
|
6
6
|
import time
|
7
|
-
from collections.abc import Mapping
|
8
|
-
from typing import
|
9
|
-
|
10
|
-
import redis
|
7
|
+
from collections.abc import Callable, Mapping
|
8
|
+
from typing import TYPE_CHECKING, Any
|
11
9
|
|
12
10
|
from c2cwsgiutils.broadcast import interface, local, utils
|
13
11
|
|
12
|
+
if TYPE_CHECKING:
|
13
|
+
import redis
|
14
|
+
|
14
15
|
_LOG = logging.getLogger(__name__)
|
15
16
|
|
16
17
|
|
@@ -33,7 +34,7 @@ class RedisBroadcaster(interface.BaseBroadcaster):
|
|
33
34
|
self._pub_sub = self._master.pubsub(ignore_subscribe_messages=True)
|
34
35
|
|
35
36
|
# Need to be subscribed to something for the thread to stay alive
|
36
|
-
self._pub_sub.subscribe(**{self._get_channel("c2c_dummy"): lambda
|
37
|
+
self._pub_sub.subscribe(**{self._get_channel("c2c_dummy"): lambda _: None})
|
37
38
|
self._thread = redis_utils.PubSubWorkerThread(self._pub_sub, name="c2c_broadcast_listener")
|
38
39
|
self._thread.start()
|
39
40
|
|
@@ -48,7 +49,7 @@ class RedisBroadcaster(interface.BaseBroadcaster):
|
|
48
49
|
data = json.loads(message["data"])
|
49
50
|
try:
|
50
51
|
response = callback(**data["params"])
|
51
|
-
except Exception as e: # pragma: no cover # pylint: disable=broad-
|
52
|
+
except Exception as e: # pragma: no cover # pylint: disable=broad-exception-caught
|
52
53
|
_LOG.error("Failed handling a broadcast message", exc_info=True)
|
53
54
|
response = {"status": 500, "message": str(e)}
|
54
55
|
answer_channel = data.get("answer_channel")
|
@@ -67,17 +68,23 @@ class RedisBroadcaster(interface.BaseBroadcaster):
|
|
67
68
|
self._pub_sub.unsubscribe(actual_channel)
|
68
69
|
|
69
70
|
def broadcast(
|
70
|
-
self,
|
71
|
-
|
71
|
+
self,
|
72
|
+
channel: str,
|
73
|
+
params: Mapping[str, Any],
|
74
|
+
expect_answers: bool,
|
75
|
+
timeout: float,
|
76
|
+
) -> list[Any] | None:
|
72
77
|
"""Broadcast a message to all the listeners."""
|
73
78
|
if expect_answers:
|
74
79
|
return self._broadcast_with_answer(channel, params, timeout)
|
75
|
-
|
76
|
-
|
77
|
-
return None
|
80
|
+
self._broadcast(channel, {"params": params})
|
81
|
+
return None
|
78
82
|
|
79
83
|
def _broadcast_with_answer(
|
80
|
-
self,
|
84
|
+
self,
|
85
|
+
channel: str,
|
86
|
+
params: Mapping[str, Any] | None,
|
87
|
+
timeout: float,
|
81
88
|
) -> list[Any]:
|
82
89
|
cond = threading.Condition()
|
83
90
|
answers = []
|
c2cwsgiutils/client_info.py
CHANGED
@@ -1,7 +1,8 @@
|
|
1
1
|
import logging
|
2
2
|
import os
|
3
3
|
import re
|
4
|
-
from
|
4
|
+
from collections.abc import Callable
|
5
|
+
from typing import Any
|
5
6
|
|
6
7
|
_LOG = logging.getLogger(__name__)
|
7
8
|
SEP_RE = re.compile(r", *")
|
@@ -15,7 +16,7 @@ class Filter:
|
|
15
16
|
Concerned headers: Forwarded and the X_Forwarded_* Headers.
|
16
17
|
"""
|
17
18
|
|
18
|
-
def __init__(self, application: Callable[[dict[str, str], Any], Any]):
|
19
|
+
def __init__(self, application: Callable[[dict[str, str], Any], Any]) -> None:
|
19
20
|
"""Initialize the filter."""
|
20
21
|
self._application = application
|
21
22
|
|
c2cwsgiutils/config_utils.py
CHANGED
@@ -1,8 +1,8 @@
|
|
1
1
|
"""Private utilities."""
|
2
2
|
|
3
3
|
import os
|
4
|
-
from collections.abc import Mapping
|
5
|
-
from typing import Any,
|
4
|
+
from collections.abc import Callable, Mapping
|
5
|
+
from typing import Any, cast
|
6
6
|
|
7
7
|
import pyramid.config
|
8
8
|
|
@@ -13,22 +13,26 @@ def get_base_path(config: pyramid.config.Configurator) -> str:
|
|
13
13
|
|
14
14
|
|
15
15
|
def env_or_config(
|
16
|
-
config:
|
17
|
-
env_name:
|
18
|
-
config_name:
|
16
|
+
config: pyramid.config.Configurator | None,
|
17
|
+
env_name: str | None = None,
|
18
|
+
config_name: str | None = None,
|
19
19
|
default: Any = None,
|
20
20
|
type_: Callable[[str], Any] = str,
|
21
21
|
) -> Any:
|
22
22
|
"""Get the setting from the environment or from the config file."""
|
23
23
|
return env_or_settings(
|
24
|
-
config.get_settings() if config is not None else {},
|
24
|
+
config.get_settings() if config is not None else {},
|
25
|
+
env_name,
|
26
|
+
config_name,
|
27
|
+
default,
|
28
|
+
type_,
|
25
29
|
)
|
26
30
|
|
27
31
|
|
28
32
|
def env_or_settings(
|
29
|
-
settings:
|
30
|
-
env_name:
|
31
|
-
settings_name:
|
33
|
+
settings: Mapping[str, Any] | None,
|
34
|
+
env_name: str | None = None,
|
35
|
+
settings_name: str | None = None,
|
32
36
|
default: Any = None,
|
33
37
|
type_: Callable[[str], Any] = str,
|
34
38
|
) -> Any:
|
@@ -40,7 +44,7 @@ def env_or_settings(
|
|
40
44
|
return default
|
41
45
|
|
42
46
|
|
43
|
-
def config_bool(value:
|
47
|
+
def config_bool(value: str | None) -> bool:
|
44
48
|
"""Get boolean from the value."""
|
45
49
|
if value is None:
|
46
50
|
return False
|
c2cwsgiutils/coverage_setup.py
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
import logging
|
2
2
|
import os
|
3
3
|
import warnings
|
4
|
-
from
|
4
|
+
from pathlib import Path
|
5
5
|
|
6
6
|
import pyramid.config
|
7
7
|
|
@@ -14,7 +14,7 @@ def init() -> None:
|
|
14
14
|
includeme()
|
15
15
|
|
16
16
|
|
17
|
-
def includeme(config:
|
17
|
+
def includeme(config: pyramid.config.Configurator | None = None) -> None:
|
18
18
|
"""Initialize the code coverage."""
|
19
19
|
del config # unused
|
20
20
|
if os.environ.get("COVERAGE", "0") != "1":
|
@@ -22,10 +22,10 @@ def includeme(config: Optional[pyramid.config.Configurator] = None) -> None:
|
|
22
22
|
import coverage # pylint: disable=import-outside-toplevel
|
23
23
|
|
24
24
|
_LOG.warning("Setting up code coverage")
|
25
|
-
report_dir = "/tmp/coverage/api" # noqa: S108 # nosec
|
26
|
-
|
25
|
+
report_dir = Path("/tmp/coverage/api") # noqa: S108 # nosec
|
26
|
+
report_dir.mkdir(parents=True, exist_ok=True)
|
27
27
|
cov = coverage.Coverage(
|
28
|
-
data_file=
|
28
|
+
data_file=str(report_dir / "coverage"),
|
29
29
|
data_suffix=True,
|
30
30
|
auto_data=True,
|
31
31
|
branch=True,
|
c2cwsgiutils/db.py
CHANGED
@@ -3,9 +3,9 @@
|
|
3
3
|
import logging
|
4
4
|
import re
|
5
5
|
import warnings
|
6
|
-
from collections.abc import Iterable
|
6
|
+
from collections.abc import Callable, Iterable
|
7
7
|
from re import Pattern
|
8
|
-
from typing import Any,
|
8
|
+
from typing import Any, cast
|
9
9
|
|
10
10
|
import pyramid.config
|
11
11
|
import pyramid.config.settings
|
@@ -37,11 +37,11 @@ tweens = Tweens()
|
|
37
37
|
def setup_session(
|
38
38
|
config: pyramid.config.Configurator,
|
39
39
|
master_prefix: str,
|
40
|
-
slave_prefix:
|
41
|
-
force_master:
|
42
|
-
force_slave:
|
40
|
+
slave_prefix: str | None = None,
|
41
|
+
force_master: Iterable[str] | None = None,
|
42
|
+
force_slave: Iterable[str] | None = None,
|
43
43
|
) -> tuple[
|
44
|
-
|
44
|
+
sqlalchemy.orm.Session | _scoped_session,
|
45
45
|
sqlalchemy.engine.Engine,
|
46
46
|
sqlalchemy.engine.Engine,
|
47
47
|
]:
|
@@ -69,13 +69,14 @@ def setup_session(
|
|
69
69
|
|
70
70
|
"""
|
71
71
|
warnings.warn(
|
72
|
-
"setup_session function is deprecated; use init and request.dbsession instead",
|
72
|
+
"setup_session function is deprecated; use init and request.dbsession instead",
|
73
|
+
stacklevel=2,
|
73
74
|
)
|
74
75
|
if slave_prefix is None:
|
75
76
|
slave_prefix = master_prefix
|
76
77
|
settings = config.registry.settings
|
77
78
|
rw_engine = sqlalchemy.engine_from_config(settings, master_prefix + ".")
|
78
|
-
rw_engine.c2c_name = master_prefix # type: ignore
|
79
|
+
rw_engine.c2c_name = master_prefix # type: ignore[attr-defined]
|
79
80
|
factory = sqlalchemy.orm.sessionmaker(bind=rw_engine)
|
80
81
|
register(factory)
|
81
82
|
db_session = sqlalchemy.orm.scoped_session(factory)
|
@@ -84,26 +85,26 @@ def setup_session(
|
|
84
85
|
if settings[master_prefix + ".url"] != settings.get(slave_prefix + ".url"):
|
85
86
|
_LOG.info("Using a slave DB for reading %s", master_prefix)
|
86
87
|
ro_engine = sqlalchemy.engine_from_config(config.get_settings(), slave_prefix + ".")
|
87
|
-
ro_engine.c2c_name = slave_prefix # type: ignore
|
88
|
+
ro_engine.c2c_name = slave_prefix # type: ignore[attr-defined]
|
88
89
|
tween_name = master_prefix.replace(".", "_")
|
89
90
|
_add_tween(config, tween_name, db_session, force_master, force_slave)
|
90
91
|
else:
|
91
92
|
ro_engine = rw_engine
|
92
93
|
|
93
|
-
db_session.c2c_rw_bind = rw_engine # type: ignore
|
94
|
-
db_session.c2c_ro_bind = ro_engine # type: ignore
|
94
|
+
db_session.c2c_rw_bind = rw_engine # type: ignore[attr-defined]
|
95
|
+
db_session.c2c_ro_bind = ro_engine # type: ignore[attr-defined]
|
95
96
|
return db_session, rw_engine, ro_engine
|
96
97
|
|
97
98
|
|
98
99
|
def create_session(
|
99
|
-
config:
|
100
|
+
config: pyramid.config.Configurator | None,
|
100
101
|
name: str,
|
101
102
|
url: str,
|
102
|
-
slave_url:
|
103
|
-
force_master:
|
104
|
-
force_slave:
|
103
|
+
slave_url: str | None = None,
|
104
|
+
force_master: Iterable[str] | None = None,
|
105
|
+
force_slave: Iterable[str] | None = None,
|
105
106
|
**engine_config: Any,
|
106
|
-
) ->
|
107
|
+
) -> sqlalchemy.orm.Session | _scoped_session:
|
107
108
|
"""
|
108
109
|
Create a SQLAlchemy session.
|
109
110
|
|
@@ -128,7 +129,8 @@ def create_session(
|
|
128
129
|
|
129
130
|
"""
|
130
131
|
warnings.warn(
|
131
|
-
"create_session function is deprecated; use init and request.dbsession instead",
|
132
|
+
"create_session function is deprecated; use init and request.dbsession instead",
|
133
|
+
stacklevel=2,
|
132
134
|
)
|
133
135
|
if slave_url is None:
|
134
136
|
slave_url = url
|
@@ -143,14 +145,14 @@ def create_session(
|
|
143
145
|
_LOG.info("Using a slave DB for reading %s", name)
|
144
146
|
ro_engine = sqlalchemy.create_engine(slave_url, **engine_config)
|
145
147
|
_add_tween(config, name, db_session, force_master, force_slave)
|
146
|
-
rw_engine.c2c_name = name + "_master" # type: ignore
|
147
|
-
ro_engine.c2c_name = name + "_slave" # type: ignore
|
148
|
+
rw_engine.c2c_name = name + "_master" # type: ignore[attr-defined]
|
149
|
+
ro_engine.c2c_name = name + "_slave" # type: ignore[attr-defined]
|
148
150
|
else:
|
149
|
-
rw_engine.c2c_name = name # type: ignore
|
151
|
+
rw_engine.c2c_name = name # type: ignore[attr-defined]
|
150
152
|
ro_engine = rw_engine
|
151
153
|
|
152
|
-
db_session.c2c_rw_bind = rw_engine # type: ignore
|
153
|
-
db_session.c2c_ro_bind = ro_engine # type: ignore
|
154
|
+
db_session.c2c_rw_bind = rw_engine # type: ignore[attr-defined]
|
155
|
+
db_session.c2c_ro_bind = ro_engine # type: ignore[attr-defined]
|
154
156
|
return db_session
|
155
157
|
|
156
158
|
|
@@ -158,8 +160,8 @@ def _add_tween(
|
|
158
160
|
config: pyramid.config.Configurator,
|
159
161
|
name: str,
|
160
162
|
db_session: _scoped_session,
|
161
|
-
force_master:
|
162
|
-
force_slave:
|
163
|
+
force_master: Iterable[str] | None,
|
164
|
+
force_slave: Iterable[str] | None,
|
163
165
|
) -> None:
|
164
166
|
master_paths: Iterable[Pattern[str]] = (
|
165
167
|
list(map(_RE_COMPILE, force_master)) if force_master is not None else []
|
@@ -169,7 +171,8 @@ def _add_tween(
|
|
169
171
|
)
|
170
172
|
|
171
173
|
def db_chooser_tween_factory(
|
172
|
-
handler: Callable[[pyramid.request.Request], Any],
|
174
|
+
handler: Callable[[pyramid.request.Request], Any],
|
175
|
+
_registry: Any,
|
173
176
|
) -> Callable[[pyramid.request.Request], Any]:
|
174
177
|
"""
|
175
178
|
Tween factory to route to a slave DB for read-only queries.
|
@@ -188,17 +191,17 @@ def _add_tween(
|
|
188
191
|
):
|
189
192
|
_LOG.debug(
|
190
193
|
"Using %s database for: %s",
|
191
|
-
db_session.c2c_ro_bind.c2c_name, # type: ignore
|
194
|
+
db_session.c2c_ro_bind.c2c_name, # type: ignore[attr-defined]
|
192
195
|
method_path,
|
193
196
|
)
|
194
|
-
session.bind = db_session.c2c_ro_bind # type: ignore
|
197
|
+
session.bind = db_session.c2c_ro_bind # type: ignore[attr-defined]
|
195
198
|
else:
|
196
199
|
_LOG.debug(
|
197
200
|
"Using %s database for: %s",
|
198
|
-
db_session.c2c_rw_bind.c2c_name, # type: ignore
|
201
|
+
db_session.c2c_rw_bind.c2c_name, # type: ignore[attr-defined]
|
199
202
|
method_path,
|
200
203
|
)
|
201
|
-
session.bind = db_session.c2c_rw_bind # type: ignore
|
204
|
+
session.bind = db_session.c2c_rw_bind # type: ignore[attr-defined]
|
202
205
|
|
203
206
|
try:
|
204
207
|
return handler(request)
|
@@ -216,11 +219,11 @@ class SessionFactory(_sessionmaker):
|
|
216
219
|
|
217
220
|
def __init__(
|
218
221
|
self,
|
219
|
-
force_master:
|
220
|
-
force_slave:
|
222
|
+
force_master: Iterable[str] | None,
|
223
|
+
force_slave: Iterable[str] | None,
|
221
224
|
ro_engine: sqlalchemy.engine.Engine,
|
222
225
|
rw_engine: sqlalchemy.engine.Engine,
|
223
|
-
):
|
226
|
+
) -> None:
|
224
227
|
"""Initialize the session factory."""
|
225
228
|
super().__init__()
|
226
229
|
self.master_paths: Iterable[Pattern[str]] = (
|
@@ -233,19 +236,22 @@ class SessionFactory(_sessionmaker):
|
|
233
236
|
def engine_name(self, readwrite: bool) -> str:
|
234
237
|
"""Get the engine name."""
|
235
238
|
if readwrite:
|
236
|
-
return cast(str, self.rw_engine.c2c_name) # type: ignore
|
237
|
-
return cast(str, self.ro_engine.c2c_name) # type: ignore
|
239
|
+
return cast(str, self.rw_engine.c2c_name) # type: ignore[attr-defined]
|
240
|
+
return cast(str, self.ro_engine.c2c_name) # type: ignore[attr-defined]
|
238
241
|
|
239
|
-
def __call__( # type: ignore
|
240
|
-
self,
|
242
|
+
def __call__( # type: ignore[override]
|
243
|
+
self,
|
244
|
+
request: pyramid.request.Request | None,
|
245
|
+
readwrite: bool | None = None,
|
246
|
+
**local_kw: Any,
|
241
247
|
) -> _scoped_session:
|
242
248
|
"""Set the engine based on the request."""
|
243
249
|
if readwrite is not None:
|
244
250
|
if readwrite and not FORCE_READONLY:
|
245
|
-
_LOG.debug("Using %s database", self.rw_engine.c2c_name) # type: ignore
|
251
|
+
_LOG.debug("Using %s database", self.rw_engine.c2c_name) # type: ignore[attr-defined]
|
246
252
|
self.configure(bind=self.rw_engine)
|
247
253
|
else:
|
248
|
-
_LOG.debug("Using %s database", self.ro_engine.c2c_name) # type: ignore
|
254
|
+
_LOG.debug("Using %s database", self.ro_engine.c2c_name) # type: ignore[attr-defined]
|
249
255
|
self.configure(bind=self.ro_engine)
|
250
256
|
else:
|
251
257
|
assert request is not None
|
@@ -258,16 +264,17 @@ class SessionFactory(_sessionmaker):
|
|
258
264
|
or any(r.match(method_path) for r in self.slave_paths)
|
259
265
|
)
|
260
266
|
):
|
261
|
-
_LOG.debug("Using %s database for: %s", self.ro_engine.c2c_name, method_path) # type: ignore
|
267
|
+
_LOG.debug("Using %s database for: %s", self.ro_engine.c2c_name, method_path) # type: ignore[attr-defined]
|
262
268
|
self.configure(bind=self.ro_engine)
|
263
269
|
else:
|
264
|
-
_LOG.debug("Using %s database for: %s", self.rw_engine.c2c_name, method_path) # type: ignore
|
270
|
+
_LOG.debug("Using %s database for: %s", self.rw_engine.c2c_name, method_path) # type: ignore[attr-defined]
|
265
271
|
self.configure(bind=self.rw_engine)
|
266
|
-
return super().__call__(**local_kw) # type: ignore
|
272
|
+
return super().__call__(**local_kw) # type: ignore[return-value]
|
267
273
|
|
268
274
|
|
269
275
|
def get_engine(
|
270
|
-
settings: pyramid.config.settings.Settings,
|
276
|
+
settings: pyramid.config.settings.Settings,
|
277
|
+
prefix: str = "sqlalchemy.",
|
271
278
|
) -> sqlalchemy.engine.Engine:
|
272
279
|
"""Get the engine from the settings."""
|
273
280
|
return engine_from_config(settings, prefix)
|
@@ -365,9 +372,9 @@ def get_tm_session_pyramid(
|
|
365
372
|
def init(
|
366
373
|
config: pyramid.config.Configurator,
|
367
374
|
master_prefix: str,
|
368
|
-
slave_prefix:
|
369
|
-
force_master:
|
370
|
-
force_slave:
|
375
|
+
slave_prefix: str | None = None,
|
376
|
+
force_master: Iterable[str] | None = None,
|
377
|
+
force_slave: Iterable[str] | None = None,
|
371
378
|
) -> SessionFactory:
|
372
379
|
"""
|
373
380
|
Initialize the database for a Pyramid app.
|
@@ -391,13 +398,13 @@ def init(
|
|
391
398
|
dbengine = settings.get("dbengine")
|
392
399
|
if not dbengine:
|
393
400
|
rw_engine = get_engine(settings, master_prefix + ".")
|
394
|
-
rw_engine.c2c_name = master_prefix # type: ignore
|
401
|
+
rw_engine.c2c_name = master_prefix # type: ignore[attr-defined]
|
395
402
|
|
396
403
|
# Setup a slave DB connection and add a tween to use it.
|
397
404
|
if slave_prefix and settings[master_prefix + ".url"] != settings.get(slave_prefix + ".url"):
|
398
405
|
_LOG.info("Using a slave DB for reading %s", master_prefix)
|
399
406
|
ro_engine = get_engine(config.get_settings(), slave_prefix + ".")
|
400
|
-
ro_engine.c2c_name = slave_prefix # type: ignore
|
407
|
+
ro_engine.c2c_name = slave_prefix # type: ignore[attr-defined]
|
401
408
|
else:
|
402
409
|
ro_engine = rw_engine
|
403
410
|
else:
|
@@ -1,7 +1,7 @@
|
|
1
1
|
import logging
|
2
2
|
import warnings
|
3
3
|
from collections.abc import Mapping
|
4
|
-
from typing import Any,
|
4
|
+
from typing import Any, cast
|
5
5
|
|
6
6
|
import pyramid.request
|
7
7
|
|
@@ -42,11 +42,10 @@ def _db_maintenance(request: pyramid.request.Request) -> Mapping[str, Any]:
|
|
42
42
|
_set_readonly(value=readonly)
|
43
43
|
_store(request.registry.settings, readonly)
|
44
44
|
return {"status": 200, "readonly": readonly}
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
return {"status": 200, "current_readonly": readonly}
|
45
|
+
readonly = _get_redis_value(request.registry.settings)
|
46
|
+
if readonly is not None:
|
47
|
+
readonly = readonly == "true"
|
48
|
+
return {"status": 200, "current_readonly": readonly}
|
50
49
|
|
51
50
|
|
52
51
|
@broadcast.decorator(expect_answers=True)
|
@@ -63,7 +62,7 @@ def _restore(config: pyramid.config.Configurator) -> None:
|
|
63
62
|
db.FORCE_READONLY = readonly == "true"
|
64
63
|
except ImportError:
|
65
64
|
pass # don't have redis
|
66
|
-
except Exception: # pylint: disable=broad-
|
65
|
+
except Exception: # pylint: disable=broad-exception-caught
|
67
66
|
# survive an error since crashing now can have bad consequences for the service. :/
|
68
67
|
_LOG.error("Cannot restore readonly DB status.", exc_info=True)
|
69
68
|
|
@@ -74,7 +73,7 @@ def _store(settings: Mapping[str, Any], readonly: bool) -> None:
|
|
74
73
|
master.set(_REDIS_PREFIX + "force_readonly", "true" if readonly else "false")
|
75
74
|
|
76
75
|
|
77
|
-
def _get_redis_value(settings: Mapping[str, Any]) ->
|
76
|
+
def _get_redis_value(settings: Mapping[str, Any]) -> str | None:
|
78
77
|
_, slave, _ = redis_utils.get(settings)
|
79
78
|
if slave is not None:
|
80
79
|
value = slave.get(_REDIS_PREFIX + "force_readonly")
|
c2cwsgiutils/debug/__init__.py
CHANGED
@@ -1,5 +1,4 @@
|
|
1
1
|
import warnings
|
2
|
-
from typing import Optional
|
3
2
|
|
4
3
|
import pyramid.config
|
5
4
|
|
@@ -29,7 +28,7 @@ def includeme(config: pyramid.config.Configurator) -> None:
|
|
29
28
|
_views.init(config)
|
30
29
|
|
31
30
|
|
32
|
-
def init_daemon(config:
|
31
|
+
def init_daemon(config: pyramid.config.Configurator | None = None) -> None:
|
33
32
|
"""
|
34
33
|
Initialize the debug broadcast listeners.
|
35
34
|
|
c2cwsgiutils/debug/_listeners.py
CHANGED
@@ -4,7 +4,7 @@ import threading
|
|
4
4
|
import time
|
5
5
|
import traceback
|
6
6
|
from collections.abc import Mapping
|
7
|
-
from typing import Any,
|
7
|
+
from typing import Any, cast
|
8
8
|
|
9
9
|
import objgraph
|
10
10
|
|
@@ -17,7 +17,7 @@ FILES_FIELDS = {"__name__", "__doc__", "__package__", "__loader__", "__spec__",
|
|
17
17
|
def _dump_stacks_impl() -> dict[str, Any]:
|
18
18
|
id2name = {th.ident: th.name for th in threading.enumerate()}
|
19
19
|
threads = {}
|
20
|
-
for thread_id, stack in sys._current_frames().items(): # pylint: disable=
|
20
|
+
for thread_id, stack in sys._current_frames().items(): # pylint: disable=protected-access
|
21
21
|
frames = []
|
22
22
|
for filename, lineno, name, line in traceback.extract_stack(stack):
|
23
23
|
cur = {"file": filename, "line": lineno, "function": name}
|
@@ -31,7 +31,7 @@ def _dump_stacks_impl() -> dict[str, Any]:
|
|
31
31
|
# pylint: disable=too-many-branches
|
32
32
|
def _dump_memory_impl(
|
33
33
|
limit: int,
|
34
|
-
analyze_type:
|
34
|
+
analyze_type: str | None,
|
35
35
|
python_internals_map: bool = False,
|
36
36
|
) -> Mapping[str, Any]:
|
37
37
|
nb_collected = [gc.collect(generation) for generation in range(3)]
|
@@ -39,7 +39,9 @@ def _dump_memory_impl(
|
|
39
39
|
"nb_collected": nb_collected,
|
40
40
|
"most_common_types": objgraph.most_common_types(limit=limit, shortnames=False),
|
41
41
|
"leaking_objects": objgraph.most_common_types(
|
42
|
-
limit=limit,
|
42
|
+
limit=limit,
|
43
|
+
shortnames=False,
|
44
|
+
objects=objgraph.get_leaking_objects(),
|
43
45
|
),
|
44
46
|
}
|
45
47
|
|