c2cwsgiutils 6.1.0.dev105__py3-none-any.whl → 6.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- c2cwsgiutils/__init__.py +14 -11
- c2cwsgiutils/acceptance/__init__.py +2 -3
- c2cwsgiutils/acceptance/connection.py +1 -2
- c2cwsgiutils/acceptance/image.py +7 -8
- c2cwsgiutils/acceptance/package-lock.json +306 -213
- c2cwsgiutils/acceptance/package.json +2 -2
- c2cwsgiutils/acceptance/print.py +7 -3
- c2cwsgiutils/acceptance/utils.py +1 -3
- c2cwsgiutils/auth.py +27 -25
- c2cwsgiutils/broadcast/__init__.py +15 -16
- c2cwsgiutils/broadcast/interface.py +3 -3
- c2cwsgiutils/broadcast/local.py +1 -0
- c2cwsgiutils/broadcast/redis.py +13 -12
- c2cwsgiutils/client_info.py +19 -1
- c2cwsgiutils/coverage_setup.py +4 -3
- c2cwsgiutils/db.py +35 -41
- c2cwsgiutils/db_maintenance_view.py +13 -13
- c2cwsgiutils/debug/__init__.py +2 -2
- c2cwsgiutils/debug/_listeners.py +1 -1
- c2cwsgiutils/debug/_views.py +7 -6
- c2cwsgiutils/debug/utils.py +9 -9
- c2cwsgiutils/errors.py +13 -15
- c2cwsgiutils/health_check.py +24 -30
- c2cwsgiutils/index.py +7 -9
- c2cwsgiutils/loader.py +1 -1
- c2cwsgiutils/logging_view.py +12 -12
- c2cwsgiutils/models_graph.py +0 -1
- c2cwsgiutils/pretty_json.py +0 -1
- c2cwsgiutils/prometheus.py +1 -7
- c2cwsgiutils/pyramid.py +0 -1
- c2cwsgiutils/pyramid_logging.py +1 -1
- c2cwsgiutils/redis_stats.py +9 -9
- c2cwsgiutils/redis_utils.py +19 -18
- c2cwsgiutils/request_tracking/__init__.py +13 -13
- c2cwsgiutils/request_tracking/_sql.py +0 -1
- c2cwsgiutils/scripts/genversion.py +5 -5
- c2cwsgiutils/scripts/stats_db.py +19 -17
- c2cwsgiutils/scripts/test_print.py +5 -5
- c2cwsgiutils/sentry.py +55 -20
- c2cwsgiutils/services.py +2 -2
- c2cwsgiutils/setup_process.py +0 -1
- c2cwsgiutils/sql_profiler/__init__.py +5 -6
- c2cwsgiutils/sql_profiler/_impl.py +18 -17
- c2cwsgiutils/sqlalchemylogger/README.md +30 -13
- c2cwsgiutils/sqlalchemylogger/handlers.py +11 -10
- c2cwsgiutils/stats_pyramid/__init__.py +1 -5
- c2cwsgiutils/stats_pyramid/_db_spy.py +2 -2
- c2cwsgiutils/stats_pyramid/_pyramid_spy.py +0 -1
- c2cwsgiutils/version.py +11 -5
- {c2cwsgiutils-6.1.0.dev105.dist-info → c2cwsgiutils-6.1.1.dist-info}/LICENSE +1 -1
- {c2cwsgiutils-6.1.0.dev105.dist-info → c2cwsgiutils-6.1.1.dist-info}/METADATA +17 -5
- c2cwsgiutils-6.1.1.dist-info/RECORD +67 -0
- {c2cwsgiutils-6.1.0.dev105.dist-info → c2cwsgiutils-6.1.1.dist-info}/WHEEL +1 -1
- c2cwsgiutils-6.1.0.dev105.dist-info/RECORD +0 -67
- {c2cwsgiutils-6.1.0.dev105.dist-info → c2cwsgiutils-6.1.1.dist-info}/entry_points.txt +0 -0
@@ -8,9 +8,9 @@ import sys
|
|
8
8
|
import warnings
|
9
9
|
from typing import Optional, cast
|
10
10
|
|
11
|
-
|
12
|
-
|
13
|
-
|
11
|
+
_SRC_VERSION_RE = re.compile(r"^.*\(([^=]*)===?([^=]*)\)$")
|
12
|
+
_VERSION_RE = re.compile(r"^([^=]*)==([^=]*)$")
|
13
|
+
_LOG = logging.getLogger(__name__)
|
14
14
|
|
15
15
|
|
16
16
|
def _get_package_version(comp: str) -> tuple[Optional[str], Optional[str]]:
|
@@ -19,8 +19,8 @@ def _get_package_version(comp: str) -> tuple[Optional[str], Optional[str]]:
|
|
19
19
|
|
20
20
|
See test_genversion.py for examples.
|
21
21
|
"""
|
22
|
-
src_matcher =
|
23
|
-
matcher = src_matcher or
|
22
|
+
src_matcher = _SRC_VERSION_RE.match(comp)
|
23
|
+
matcher = src_matcher or _VERSION_RE.match(comp)
|
24
24
|
if matcher:
|
25
25
|
return cast(tuple[str, str], matcher.groups())
|
26
26
|
else:
|
c2cwsgiutils/scripts/stats_db.py
CHANGED
@@ -6,7 +6,7 @@ import logging
|
|
6
6
|
import os
|
7
7
|
import sys
|
8
8
|
import time
|
9
|
-
from typing import
|
9
|
+
from typing import Optional
|
10
10
|
from wsgiref.simple_server import make_server
|
11
11
|
|
12
12
|
import sqlalchemy
|
@@ -20,12 +20,9 @@ from zope.sqlalchemy import register
|
|
20
20
|
import c2cwsgiutils.setup_process
|
21
21
|
from c2cwsgiutils import prometheus
|
22
22
|
|
23
|
-
|
24
|
-
scoped_session = sqlalchemy.orm.scoped_session[sqlalchemy.orm.Session]
|
25
|
-
else:
|
26
|
-
scoped_session = sqlalchemy.orm.scoped_session
|
23
|
+
scoped_session = sqlalchemy.orm.scoped_session[sqlalchemy.orm.Session]
|
27
24
|
|
28
|
-
|
25
|
+
_LOG = logging.getLogger(__name__)
|
29
26
|
|
30
27
|
|
31
28
|
def _parse_args() -> argparse.Namespace:
|
@@ -72,6 +69,7 @@ class Reporter:
|
|
72
69
|
self.gauges: dict[str, Gauge] = {}
|
73
70
|
|
74
71
|
def get_gauge(self, kind: str, kind_help: str, labels: list[str]) -> Gauge:
|
72
|
+
"""Get a gauge."""
|
75
73
|
if kind not in self.gauges:
|
76
74
|
self.gauges[kind] = Gauge(
|
77
75
|
prometheus.build_metric_name(f"database_{kind}"),
|
@@ -84,25 +82,30 @@ class Reporter:
|
|
84
82
|
def do_report(
|
85
83
|
self, metric: list[str], value: int, kind: str, kind_help: str, tags: dict[str, str]
|
86
84
|
) -> None:
|
87
|
-
|
85
|
+
"""Report a metric."""
|
86
|
+
_LOG.debug("%s.%s -> %d", kind, ".".join(metric), value)
|
88
87
|
gauge = self.get_gauge(kind, kind_help, list(tags.keys()))
|
89
88
|
gauge.labels(**tags).set(value)
|
90
89
|
|
91
90
|
def commit(self) -> None:
|
91
|
+
"""Commit the metrics."""
|
92
92
|
if self.prometheus_push:
|
93
93
|
push_to_gateway(self.args.prometheus_url, job="db_counts", registry=self.registry)
|
94
94
|
else:
|
95
95
|
port = int(os.environ.get("C2C_PROMETHEUS_PORT", "9090"))
|
96
96
|
app = make_wsgi_app(self.registry)
|
97
97
|
with make_server("", port, app) as httpd:
|
98
|
-
|
98
|
+
_LOG.info("Waiting that Prometheus get the metrics served on port %s...", port)
|
99
99
|
httpd.handle_request()
|
100
100
|
|
101
101
|
def error(self, metric: list[str], error_: Exception) -> None:
|
102
|
+
"""Report an error."""
|
103
|
+
del metric
|
102
104
|
if self._error is None:
|
103
105
|
self._error = error_
|
104
106
|
|
105
107
|
def report_error(self) -> None:
|
108
|
+
"""Raise the error if any."""
|
106
109
|
if self._error is not None:
|
107
110
|
raise self._error
|
108
111
|
|
@@ -225,7 +228,6 @@ def _do_table_count(
|
|
225
228
|
|
226
229
|
def do_extra(session: scoped_session, sql: str, kind: str, gauge_help: str, reporter: Reporter) -> None:
|
227
230
|
"""Do an extra report."""
|
228
|
-
|
229
231
|
for metric, count in session.execute(sqlalchemy.text(sql)):
|
230
232
|
reporter.do_report(
|
231
233
|
str(metric).split("."), count, kind=kind, kind_help=gauge_help, tags={"metric": metric}
|
@@ -253,29 +255,29 @@ def _do_dtats_db(args: argparse.Namespace) -> None:
|
|
253
255
|
params={"schemas": tuple(args.schema)},
|
254
256
|
).fetchall()
|
255
257
|
for schema, table in tables:
|
256
|
-
|
258
|
+
_LOG.info("Process table %s.%s.", schema, table)
|
257
259
|
try:
|
258
260
|
do_table(session, schema, table, reporter)
|
259
261
|
except Exception as e: # pylint: disable=broad-except
|
260
|
-
|
262
|
+
_LOG.exception("Process table %s.%s error.", schema, table)
|
261
263
|
reporter.error([schema, table], e)
|
262
264
|
|
263
265
|
if args.extra:
|
264
266
|
for pos, extra in enumerate(args.extra):
|
265
|
-
|
267
|
+
_LOG.info("Process extra %s.", extra)
|
266
268
|
try:
|
267
269
|
do_extra(session, extra, "extra", "Extra metric", reporter)
|
268
270
|
except Exception as e: # pylint: disable=broad-except
|
269
|
-
|
271
|
+
_LOG.exception("Process extra %s error.", extra)
|
270
272
|
reporter.error(["extra", str(pos + 1)], e)
|
271
273
|
if args.extra_gauge:
|
272
274
|
for pos, extra in enumerate(args.extra_gauge):
|
273
275
|
sql, gauge, gauge_help = extra
|
274
|
-
|
276
|
+
_LOG.info("Process extra %s.", extra)
|
275
277
|
try:
|
276
278
|
do_extra(session, sql, gauge, gauge_help, reporter)
|
277
279
|
except Exception as e: # pylint: disable=broad-except
|
278
|
-
|
280
|
+
_LOG.exception("Process extra %s error.", extra)
|
279
281
|
reporter.error(["extra", str(len(args.extra) + pos + 1)], e)
|
280
282
|
|
281
283
|
reporter.commit()
|
@@ -294,11 +296,11 @@ def main() -> None:
|
|
294
296
|
success = True
|
295
297
|
break
|
296
298
|
except: # pylint: disable=bare-except
|
297
|
-
|
299
|
+
_LOG.exception("Exception during run")
|
298
300
|
time.sleep(float(os.environ.get("C2CWSGIUTILS_STATS_DB_SLEEP", 1)))
|
299
301
|
|
300
302
|
if not success:
|
301
|
-
|
303
|
+
_LOG.error("Not in success, exiting")
|
302
304
|
sys.exit(1)
|
303
305
|
|
304
306
|
|
@@ -8,7 +8,7 @@ import warnings
|
|
8
8
|
import c2cwsgiutils.setup_process
|
9
9
|
from c2cwsgiutils.acceptance.print import PrintConnection
|
10
10
|
|
11
|
-
|
11
|
+
_LOG = logging.getLogger(__name__)
|
12
12
|
|
13
13
|
|
14
14
|
def _parse_args() -> argparse.Namespace:
|
@@ -38,7 +38,7 @@ def main() -> None:
|
|
38
38
|
if args.app is None:
|
39
39
|
for app in print_.get_apps():
|
40
40
|
if app != "default":
|
41
|
-
|
41
|
+
_LOG.info("\n\n%s=================", app)
|
42
42
|
test_app(print_, app)
|
43
43
|
else:
|
44
44
|
test_app(print_, args.app)
|
@@ -47,13 +47,13 @@ def main() -> None:
|
|
47
47
|
def test_app(print_: PrintConnection, app: str) -> None:
|
48
48
|
"""Test the application."""
|
49
49
|
capabilities = print_.get_capabilities(app)
|
50
|
-
|
50
|
+
_LOG.debug("Capabilities:\n%s", pprint.pformat(capabilities))
|
51
51
|
examples = print_.get_example_requests(app)
|
52
52
|
for name, request in examples.items():
|
53
|
-
|
53
|
+
_LOG.info("\n%s-----------------", name)
|
54
54
|
pdf = print_.get_pdf(app, request)
|
55
55
|
size = len(pdf.content)
|
56
|
-
|
56
|
+
_LOG.info("Size=%d", size)
|
57
57
|
|
58
58
|
|
59
59
|
if __name__ == "__main__":
|
c2cwsgiutils/sentry.py
CHANGED
@@ -6,7 +6,8 @@ from collections.abc import Generator, MutableMapping
|
|
6
6
|
from typing import Any, Callable, Optional
|
7
7
|
|
8
8
|
import pyramid.config
|
9
|
-
import sentry_sdk
|
9
|
+
import sentry_sdk.integrations
|
10
|
+
from sentry_sdk.integrations.asyncio import AsyncioIntegration
|
10
11
|
from sentry_sdk.integrations.logging import LoggingIntegration, ignore_logger
|
11
12
|
from sentry_sdk.integrations.pyramid import PyramidIntegration
|
12
13
|
from sentry_sdk.integrations.redis import RedisIntegration
|
@@ -15,14 +16,15 @@ from sentry_sdk.integrations.wsgi import SentryWsgiMiddleware
|
|
15
16
|
|
16
17
|
from c2cwsgiutils import config_utils
|
17
18
|
|
18
|
-
|
19
|
-
|
19
|
+
_LOG = logging.getLogger(__name__)
|
20
|
+
_CLIENT_SETUP = False
|
20
21
|
|
21
22
|
|
22
23
|
def _create_before_send_filter(tags: MutableMapping[str, str]) -> Callable[[Any, Any], Any]:
|
23
24
|
"""Create a filter that adds tags to every events."""
|
24
25
|
|
25
26
|
def do_filter(event: Any, hint: Any) -> Any:
|
27
|
+
del hint
|
26
28
|
event.setdefault("tags", {}).update(tags)
|
27
29
|
return event
|
28
30
|
|
@@ -31,17 +33,15 @@ def _create_before_send_filter(tags: MutableMapping[str, str]) -> Callable[[Any,
|
|
31
33
|
|
32
34
|
def init(config: Optional[pyramid.config.Configurator] = None) -> None:
|
33
35
|
"""Initialize the Sentry integration, for backward compatibility."""
|
34
|
-
|
35
36
|
warnings.warn("init function is deprecated; use includeme instead")
|
36
37
|
includeme(config)
|
37
38
|
|
38
39
|
|
39
40
|
def includeme(config: Optional[pyramid.config.Configurator] = None) -> None:
|
40
41
|
"""Initialize the Sentry integration."""
|
41
|
-
|
42
|
-
global _client_setup
|
42
|
+
global _CLIENT_SETUP # pylint: disable=global-statement
|
43
43
|
sentry_url = config_utils.env_or_config(config, "SENTRY_URL", "c2c.sentry.url")
|
44
|
-
if sentry_url is not None and not
|
44
|
+
if sentry_url is not None and not _CLIENT_SETUP:
|
45
45
|
client_info: MutableMapping[str, Any] = {
|
46
46
|
key[14:].lower(): value for key, value in os.environ.items() if key.startswith("SENTRY_CLIENT_")
|
47
47
|
}
|
@@ -55,6 +55,7 @@ def includeme(config: Optional[pyramid.config.Configurator] = None) -> None:
|
|
55
55
|
"propagate_traces",
|
56
56
|
"auto_enabling_integrations",
|
57
57
|
"auto_session_tracking",
|
58
|
+
"enable_tracing",
|
58
59
|
):
|
59
60
|
if key in client_info:
|
60
61
|
client_info[key] = client_info[key].lower() in ("1", "t", "true")
|
@@ -73,25 +74,58 @@ def includeme(config: Optional[pyramid.config.Configurator] = None) -> None:
|
|
73
74
|
client_info["ignore_errors"] = client_info.pop("ignore_exceptions", "SystemExit").split(",")
|
74
75
|
tags = {key[11:].lower(): value for key, value in os.environ.items() if key.startswith("SENTRY_TAG_")}
|
75
76
|
|
76
|
-
sentry_logging = LoggingIntegration(
|
77
|
-
level=logging.DEBUG,
|
78
|
-
event_level=config_utils.env_or_config(
|
79
|
-
config, "SENTRY_LEVEL", "c2c.sentry_level", "ERROR"
|
80
|
-
).upper(),
|
81
|
-
)
|
82
77
|
traces_sample_rate = float(
|
83
78
|
config_utils.env_or_config(
|
84
79
|
config, "SENTRY_TRACES_SAMPLE_RATE", "c2c.sentry_traces_sample_rate", "0.0"
|
85
80
|
)
|
86
81
|
)
|
82
|
+
integrations: list[sentry_sdk.integrations.Integration] = []
|
83
|
+
if config_utils.config_bool(
|
84
|
+
config_utils.env_or_config(
|
85
|
+
config, "SENTRY_INTEGRATION_LOGGING", "c2c.sentry_integration_logging", "true"
|
86
|
+
)
|
87
|
+
):
|
88
|
+
integrations.append(
|
89
|
+
LoggingIntegration(
|
90
|
+
level=logging.DEBUG,
|
91
|
+
event_level=config_utils.env_or_config(
|
92
|
+
config, "SENTRY_LEVEL", "c2c.sentry_level", "ERROR"
|
93
|
+
).upper(),
|
94
|
+
)
|
95
|
+
)
|
96
|
+
if config_utils.config_bool(
|
97
|
+
config_utils.env_or_config(
|
98
|
+
config, "SENTRY_INTEGRATION_PYRAMID", "c2c.sentry_integration_pyramid", "true"
|
99
|
+
)
|
100
|
+
):
|
101
|
+
integrations.append(PyramidIntegration())
|
102
|
+
if config_utils.config_bool(
|
103
|
+
config_utils.env_or_config(
|
104
|
+
config, "SENTRY_INTEGRATION_SQLALCHEMY", "c2c.sentry_integration_sqlalchemy", "true"
|
105
|
+
)
|
106
|
+
):
|
107
|
+
integrations.append(SqlalchemyIntegration())
|
108
|
+
if config_utils.config_bool(
|
109
|
+
config_utils.env_or_config(
|
110
|
+
config, "SENTRY_INTEGRATION_REDIS", "c2c.sentry_integration_redis", "true"
|
111
|
+
)
|
112
|
+
):
|
113
|
+
integrations.append(RedisIntegration())
|
114
|
+
if config_utils.config_bool(
|
115
|
+
config_utils.env_or_config(
|
116
|
+
config, "SENTRY_INTEGRATION_ASYNCIO", "c2c.sentry_integration_asyncio", "true"
|
117
|
+
)
|
118
|
+
):
|
119
|
+
integrations.append(AsyncioIntegration())
|
120
|
+
|
87
121
|
sentry_sdk.init(
|
88
122
|
dsn=sentry_url,
|
89
|
-
integrations=
|
123
|
+
integrations=integrations,
|
90
124
|
traces_sample_rate=traces_sample_rate,
|
91
125
|
before_send=_create_before_send_filter(tags),
|
92
126
|
**client_info,
|
93
127
|
)
|
94
|
-
|
128
|
+
_CLIENT_SETUP = True
|
95
129
|
|
96
130
|
excludes = config_utils.env_or_config(
|
97
131
|
config, "SENTRY_EXCLUDES", "c2c.sentry.excludes", "sentry_sdk"
|
@@ -99,7 +133,7 @@ def includeme(config: Optional[pyramid.config.Configurator] = None) -> None:
|
|
99
133
|
for exclude in excludes:
|
100
134
|
ignore_logger(exclude)
|
101
135
|
|
102
|
-
|
136
|
+
_LOG.info("Configured sentry reporting with client=%s and tags=%s", repr(client_info), repr(tags))
|
103
137
|
|
104
138
|
|
105
139
|
@contextlib.contextmanager
|
@@ -110,7 +144,7 @@ def capture_exceptions() -> Generator[None, None, None]:
|
|
110
144
|
You don't need to use that for exception terminating the process (those not caught). Sentry does that
|
111
145
|
already.
|
112
146
|
"""
|
113
|
-
if
|
147
|
+
if _CLIENT_SETUP:
|
114
148
|
try:
|
115
149
|
yield
|
116
150
|
except Exception:
|
@@ -122,12 +156,12 @@ def capture_exceptions() -> Generator[None, None, None]:
|
|
122
156
|
|
123
157
|
def filter_wsgi_app(application: Callable[..., Any]) -> Callable[..., Any]:
|
124
158
|
"""If sentry is configured, add a Sentry filter around the application."""
|
125
|
-
if
|
159
|
+
if _CLIENT_SETUP:
|
126
160
|
try:
|
127
|
-
|
161
|
+
_LOG.info("Enable WSGI filter for Sentry")
|
128
162
|
return SentryWsgiMiddleware(application)
|
129
163
|
except Exception: # pylint: disable=broad-except
|
130
|
-
|
164
|
+
_LOG.error("Failed enabling sentry. Continuing without it.", exc_info=True)
|
131
165
|
return application
|
132
166
|
else:
|
133
167
|
return application
|
@@ -135,4 +169,5 @@ def filter_wsgi_app(application: Callable[..., Any]) -> Callable[..., Any]:
|
|
135
169
|
|
136
170
|
def filter_factory(*args: Any, **kwargs: Any) -> Callable[..., Any]:
|
137
171
|
"""Get the filter."""
|
172
|
+
del args, kwargs
|
138
173
|
return filter_wsgi_app
|
c2cwsgiutils/services.py
CHANGED
@@ -5,7 +5,7 @@ from cornice import Service
|
|
5
5
|
from pyramid.request import Request
|
6
6
|
from pyramid.response import Response
|
7
7
|
|
8
|
-
|
8
|
+
_LOG = logging.getLogger(__name__)
|
9
9
|
|
10
10
|
|
11
11
|
def create(name: str, path: str, *args: Any, **kwargs: Any) -> Service:
|
@@ -31,6 +31,6 @@ def _cache_cors(response: Response, request: Request) -> Response:
|
|
31
31
|
except Exception:
|
32
32
|
# cornice catches exceptions from filters, and tries call back the filter with only the request.
|
33
33
|
# This leads to a useless message in case of error...
|
34
|
-
|
34
|
+
_LOG.error("Failed fixing cache headers for CORS", exc_info=True)
|
35
35
|
raise
|
36
36
|
return response
|
c2cwsgiutils/setup_process.py
CHANGED
@@ -11,10 +11,9 @@ import pyramid.request
|
|
11
11
|
|
12
12
|
from c2cwsgiutils import auth
|
13
13
|
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
repository = None
|
14
|
+
_ENV_KEY = "C2C_SQL_PROFILER_ENABLED"
|
15
|
+
_CONFIG_KEY = "c2c.sql_profiler_enabled"
|
16
|
+
_LOG = logging.getLogger(__name__)
|
18
17
|
|
19
18
|
|
20
19
|
def init(config: pyramid.config.Configurator) -> None:
|
@@ -25,7 +24,7 @@ def init(config: pyramid.config.Configurator) -> None:
|
|
25
24
|
|
26
25
|
def includeme(config: pyramid.config.Configurator) -> None:
|
27
26
|
"""Install a pyramid event handler that adds the request information."""
|
28
|
-
if auth.is_enabled(config,
|
29
|
-
from . import _impl
|
27
|
+
if auth.is_enabled(config, _ENV_KEY, _CONFIG_KEY):
|
28
|
+
from . import _impl # pylint: disable=import-outside-toplevel
|
30
29
|
|
31
30
|
_impl.init(config)
|
@@ -16,8 +16,8 @@ import sqlalchemy.event
|
|
16
16
|
|
17
17
|
from c2cwsgiutils import auth, broadcast, config_utils
|
18
18
|
|
19
|
-
|
20
|
-
|
19
|
+
_LOG = logging.getLogger(__name__)
|
20
|
+
_REPOSITORY = None
|
21
21
|
|
22
22
|
|
23
23
|
class _Repository:
|
@@ -35,7 +35,8 @@ class _Repository:
|
|
35
35
|
_context: Any,
|
36
36
|
_executemany: Any,
|
37
37
|
) -> None:
|
38
|
-
|
38
|
+
"""Profile the SQL statement."""
|
39
|
+
if statement.startswith("SELECT ") and _LOG.isEnabledFor(logging.INFO):
|
39
40
|
do_it = False
|
40
41
|
with self._lock:
|
41
42
|
if statement not in self._repo:
|
@@ -43,8 +44,8 @@ class _Repository:
|
|
43
44
|
self._repo.add(statement)
|
44
45
|
if do_it:
|
45
46
|
try:
|
46
|
-
|
47
|
-
|
47
|
+
_LOG.info("statement:\n%s", _indent(_beautify_sql(statement)))
|
48
|
+
_LOG.info("parameters: %s", repr(parameters))
|
48
49
|
with conn.engine.begin() as c:
|
49
50
|
output = "\n ".join(
|
50
51
|
[
|
@@ -54,7 +55,7 @@ class _Repository:
|
|
54
55
|
)
|
55
56
|
]
|
56
57
|
)
|
57
|
-
|
58
|
+
_LOG.info(output)
|
58
59
|
except Exception: # nosec # pylint: disable=broad-except
|
59
60
|
pass
|
60
61
|
|
@@ -64,21 +65,21 @@ def _sql_profiler_view(request: pyramid.request.Request) -> Mapping[str, Any]:
|
|
64
65
|
enable = request.params.get("enable")
|
65
66
|
if enable is not None:
|
66
67
|
broadcast.broadcast("c2c_sql_profiler", params={"enable": enable}, expect_answers=True)
|
67
|
-
return {"status": 200, "enabled":
|
68
|
+
return {"status": 200, "enabled": _REPOSITORY is not None}
|
68
69
|
|
69
70
|
|
70
71
|
def _setup_profiler(enable: str) -> None:
|
71
|
-
global
|
72
|
+
global _REPOSITORY # pylint: disable=global-statement
|
72
73
|
if config_utils.config_bool(enable):
|
73
|
-
if
|
74
|
-
|
75
|
-
|
76
|
-
sqlalchemy.event.listen(sqlalchemy.engine.Engine, "before_cursor_execute",
|
74
|
+
if _REPOSITORY is None:
|
75
|
+
_LOG.info("Enabling the SQL profiler")
|
76
|
+
_REPOSITORY = _Repository()
|
77
|
+
sqlalchemy.event.listen(sqlalchemy.engine.Engine, "before_cursor_execute", _REPOSITORY.profile)
|
77
78
|
else:
|
78
|
-
if
|
79
|
-
|
80
|
-
sqlalchemy.event.remove(sqlalchemy.engine.Engine, "before_cursor_execute",
|
81
|
-
|
79
|
+
if _REPOSITORY is not None:
|
80
|
+
_LOG.info("Disabling the SQL profiler")
|
81
|
+
sqlalchemy.event.remove(sqlalchemy.engine.Engine, "before_cursor_execute", _REPOSITORY.profile)
|
82
|
+
_REPOSITORY = None
|
82
83
|
|
83
84
|
|
84
85
|
def _beautify_sql(statement: str) -> str:
|
@@ -102,4 +103,4 @@ def init(config: pyramid.config.Configurator) -> None:
|
|
102
103
|
"c2c_sql_profiler", config_utils.get_base_path(config) + r"/sql_profiler", request_method="GET"
|
103
104
|
)
|
104
105
|
config.add_view(_sql_profiler_view, route_name="c2c_sql_profiler", renderer="fast_json", http_cache=0)
|
105
|
-
|
106
|
+
_LOG.info("Enabled the /sql_profiler API")
|
@@ -2,22 +2,39 @@ This module is used to ship logging records to an SQL database.
|
|
2
2
|
|
3
3
|
Currently only `sqlite` and `postgres_psycopg2` are fully supported.
|
4
4
|
|
5
|
-
To add the
|
5
|
+
To add the handler, setup it directly in your app's main function. You
|
6
|
+
can add it to an existing logger (setup in you `.ini` file),
|
7
|
+
or create a new logger by calling the `logging.getlogger` method.
|
6
8
|
|
9
|
+
```python
|
10
|
+
import logging
|
11
|
+
from c2cwsgiutils.sqlalchemylogger.handlers import SQLAlchemyHandler
|
12
|
+
|
13
|
+
def _setup_sqlalchemy_logger():
|
14
|
+
"""
|
15
|
+
Setup sqlalchemy logger.
|
16
|
+
"""
|
17
|
+
logger = logging.getLogger("A_LOGGER")
|
18
|
+
handler = SQLAlchemyHandler(
|
19
|
+
sqlalchemy_url={
|
20
|
+
# "url": "sqlite:///logger_db.sqlite3",
|
21
|
+
"url": "postgresql://postgres:password@localhost:5432/test",
|
22
|
+
"tablename": "test",
|
23
|
+
"tableargs": {"schema": "xyz"},
|
24
|
+
},
|
25
|
+
does_not_contain_expression="curl",
|
26
|
+
)
|
27
|
+
logger.addHandler(handler)
|
28
|
+
|
29
|
+
def main(_, **settings):
|
30
|
+
_setup_sqlalchemy_logger ()
|
31
|
+
...
|
7
32
|
```
|
8
|
-
[handlers]
|
9
|
-
keys = sqlalchemy_logger
|
10
|
-
|
11
|
-
[handler_sqlalchemy_logger]
|
12
|
-
class = c2cwsgiutils.sqlalchemylogger.handlers.SQLAlchemyHandler
|
13
|
-
#args = ({'url':'sqlite:///logger_db.sqlite3','tablename':'test'},'curl')
|
14
|
-
args = ({'url':'postgresql://postgres:password@localhost:5432/test','tablename':'test','tableargs': {'schema':'xyz'}},'curl')
|
15
|
-
level = NOTSET
|
16
|
-
formatter = generic
|
17
|
-
propagate = 0
|
18
|
-
```
|
19
33
|
|
20
|
-
|
34
|
+
Do not set up this sqlalchemy logger in you `.ini` file directly.
|
35
|
+
It won't work (multi process issue).
|
36
|
+
|
37
|
+
if the given credentials are sufficient, the handler will
|
21
38
|
create the DB, schema and table it needs directly.
|
22
39
|
|
23
40
|
In the above example the second parameter provided `'curl'` is a negative
|
@@ -14,7 +14,7 @@ from sqlalchemy_utils import create_database, database_exists
|
|
14
14
|
from c2cwsgiutils.sqlalchemylogger._filters import ContainsExpression, DoesNotContainExpression
|
15
15
|
from c2cwsgiutils.sqlalchemylogger._models import Base, create_log_class
|
16
16
|
|
17
|
-
|
17
|
+
_LOG = logging.getLogger(__name__)
|
18
18
|
|
19
19
|
|
20
20
|
class SQLAlchemyHandler(logging.Handler):
|
@@ -30,28 +30,28 @@ class SQLAlchemyHandler(logging.Handler):
|
|
30
30
|
contains_expression: str = "",
|
31
31
|
) -> None:
|
32
32
|
super().__init__()
|
33
|
-
#
|
33
|
+
# Initialize DB session
|
34
34
|
self.engine = create_engine(sqlalchemy_url["url"])
|
35
|
-
self.Log = create_log_class(
|
35
|
+
self.Log = create_log_class( # pylint: disable=invalid-name
|
36
36
|
tablename=sqlalchemy_url.get("tablename", "logs"),
|
37
37
|
tableargs=sqlalchemy_url.get("tableargs", None), # type: ignore
|
38
38
|
)
|
39
39
|
Base.metadata.bind = self.engine
|
40
40
|
self.session = sessionmaker(bind=self.engine)() # noqa
|
41
|
-
#
|
41
|
+
# Initialize log queue
|
42
42
|
self.log_queue: Any = queue.Queue()
|
43
|
-
#
|
43
|
+
# Initialize a thread to process the logs Asynchronously
|
44
44
|
self.condition = threading.Condition()
|
45
45
|
self.processor_thread = threading.Thread(target=self._processor, daemon=True)
|
46
46
|
self.processor_thread.start()
|
47
|
-
#
|
47
|
+
# Initialize filters
|
48
48
|
if does_not_contain_expression:
|
49
49
|
self.addFilter(DoesNotContainExpression(does_not_contain_expression))
|
50
50
|
if contains_expression:
|
51
51
|
self.addFilter(ContainsExpression(contains_expression))
|
52
52
|
|
53
53
|
def _processor(self) -> None:
|
54
|
-
|
54
|
+
_LOG.debug("%s: starting processor thread", __name__)
|
55
55
|
while True:
|
56
56
|
logs = []
|
57
57
|
time_since_last = time.perf_counter()
|
@@ -70,7 +70,7 @@ class SQLAlchemyHandler(logging.Handler):
|
|
70
70
|
):
|
71
71
|
self._write_logs(logs)
|
72
72
|
break
|
73
|
-
|
73
|
+
_LOG.debug("%s: stopping processor thread", __name__)
|
74
74
|
|
75
75
|
def _write_logs(self, logs: list[Any]) -> None:
|
76
76
|
try:
|
@@ -85,12 +85,13 @@ class SQLAlchemyHandler(logging.Handler):
|
|
85
85
|
except Exception as e: # pylint: disable=broad-except
|
86
86
|
# if we really cannot commit the log to DB, do not lock the
|
87
87
|
# thread and do not crash the application
|
88
|
-
|
88
|
+
_LOG.critical(e)
|
89
89
|
finally:
|
90
90
|
self.session.expunge_all()
|
91
91
|
|
92
92
|
def create_db(self) -> None:
|
93
|
-
|
93
|
+
"""Create the database if it does not exist."""
|
94
|
+
_LOG.info("%s: creating new database", __name__)
|
94
95
|
if not database_exists(self.engine.url):
|
95
96
|
create_database(self.engine.url)
|
96
97
|
# FIXME: we should not access directly the private __table_args__
|
@@ -10,7 +10,6 @@ from c2cwsgiutils.stats_pyramid import _pyramid_spy
|
|
10
10
|
|
11
11
|
def init(config: pyramid.config.Configurator) -> None:
|
12
12
|
"""Initialize the whole stats module, for backward compatibility."""
|
13
|
-
|
14
13
|
warnings.warn("init function is deprecated; use includeme instead")
|
15
14
|
includeme(config)
|
16
15
|
|
@@ -20,17 +19,14 @@ def includeme(config: pyramid.config.Configurator) -> None:
|
|
20
19
|
Initialize the whole stats pyramid module.
|
21
20
|
|
22
21
|
Arguments:
|
23
|
-
|
24
22
|
config: The Pyramid config
|
25
23
|
"""
|
26
|
-
|
27
24
|
_pyramid_spy.init(config)
|
28
25
|
init_db_spy()
|
29
26
|
|
30
27
|
|
31
28
|
def init_db_spy() -> None:
|
32
29
|
"""Initialize the database spy."""
|
33
|
-
|
34
|
-
from . import _db_spy
|
30
|
+
from . import _db_spy # pylint: disable=import-outside-toplevel
|
35
31
|
|
36
32
|
_db_spy.init()
|
@@ -10,7 +10,7 @@ from sqlalchemy.orm import Session
|
|
10
10
|
|
11
11
|
from c2cwsgiutils import prometheus
|
12
12
|
|
13
|
-
|
13
|
+
_LOG = logging.getLogger(__name__)
|
14
14
|
|
15
15
|
_PROMETHEUS_DB_SUMMARY = prometheus_client.Summary(
|
16
16
|
prometheus.build_metric_name("database"),
|
@@ -71,7 +71,7 @@ def _create_sqlalchemy_timer_cb(what: str) -> Callable[..., Any]:
|
|
71
71
|
|
72
72
|
def after(*_args: Any, **_kwargs: Any) -> None:
|
73
73
|
_PROMETHEUS_DB_SUMMARY.labels({"query": what}).observe(time.perf_counter() - start)
|
74
|
-
|
74
|
+
_LOG.debug("Execute statement '%s' in %d.", what, time.perf_counter() - start)
|
75
75
|
|
76
76
|
return after
|
77
77
|
|
@@ -86,7 +86,6 @@ def init(config: pyramid.config.Configurator) -> None: # pragma: nocover
|
|
86
86
|
Subscribe to Pyramid events in order to get some stats on route time execution.
|
87
87
|
|
88
88
|
Arguments:
|
89
|
-
|
90
89
|
config: The Pyramid config
|
91
90
|
"""
|
92
91
|
config.add_subscriber(_request_callback, pyramid.events.NewRequest)
|