c2cwsgiutils 6.2.0.dev72__py3-none-any.whl → 6.2.0.dev75__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- c2cwsgiutils/__init__.py +4 -4
- c2cwsgiutils/acceptance/__init__.py +4 -1
- c2cwsgiutils/acceptance/connection.py +45 -32
- c2cwsgiutils/acceptance/image.py +45 -39
- c2cwsgiutils/acceptance/print.py +2 -2
- c2cwsgiutils/acceptance/utils.py +4 -4
- c2cwsgiutils/auth.py +20 -13
- c2cwsgiutils/broadcast/__init__.py +25 -16
- c2cwsgiutils/broadcast/interface.py +8 -4
- c2cwsgiutils/broadcast/local.py +9 -4
- c2cwsgiutils/broadcast/redis.py +19 -12
- c2cwsgiutils/client_info.py +3 -2
- c2cwsgiutils/config_utils.py +14 -10
- c2cwsgiutils/coverage_setup.py +5 -5
- c2cwsgiutils/db.py +54 -47
- c2cwsgiutils/db_maintenance_view.py +7 -8
- c2cwsgiutils/debug/__init__.py +1 -2
- c2cwsgiutils/debug/_listeners.py +6 -4
- c2cwsgiutils/debug/_views.py +15 -10
- c2cwsgiutils/debug/utils.py +5 -5
- c2cwsgiutils/errors.py +15 -10
- c2cwsgiutils/health_check.py +79 -60
- c2cwsgiutils/index.py +35 -37
- c2cwsgiutils/loader.py +5 -4
- c2cwsgiutils/logging_view.py +13 -6
- c2cwsgiutils/models_graph.py +5 -5
- c2cwsgiutils/pretty_json.py +3 -2
- c2cwsgiutils/profiler.py +1 -2
- c2cwsgiutils/prometheus.py +6 -6
- c2cwsgiutils/pyramid.py +1 -1
- c2cwsgiutils/pyramid_logging.py +9 -4
- c2cwsgiutils/redis_stats.py +12 -7
- c2cwsgiutils/redis_utils.py +18 -10
- c2cwsgiutils/request_tracking/__init__.py +20 -12
- c2cwsgiutils/request_tracking/_sql.py +2 -1
- c2cwsgiutils/scripts/genversion.py +10 -9
- c2cwsgiutils/scripts/stats_db.py +29 -13
- c2cwsgiutils/sentry.py +44 -20
- c2cwsgiutils/setup_process.py +7 -4
- c2cwsgiutils/sql_profiler/_impl.py +12 -11
- c2cwsgiutils/sqlalchemylogger/_models.py +3 -3
- c2cwsgiutils/sqlalchemylogger/examples/__init__.py +0 -0
- c2cwsgiutils/sqlalchemylogger/handlers.py +8 -9
- c2cwsgiutils/stats_pyramid/_db_spy.py +12 -3
- c2cwsgiutils/stats_pyramid/_pyramid_spy.py +10 -9
- c2cwsgiutils/version.py +10 -7
- {c2cwsgiutils-6.2.0.dev72.dist-info → c2cwsgiutils-6.2.0.dev75.dist-info}/METADATA +2 -2
- c2cwsgiutils-6.2.0.dev75.dist-info/RECORD +68 -0
- c2cwsgiutils-6.2.0.dev72.dist-info/RECORD +0 -67
- {c2cwsgiutils-6.2.0.dev72.dist-info → c2cwsgiutils-6.2.0.dev75.dist-info}/LICENSE +0 -0
- {c2cwsgiutils-6.2.0.dev72.dist-info → c2cwsgiutils-6.2.0.dev75.dist-info}/WHEEL +0 -0
- {c2cwsgiutils-6.2.0.dev72.dist-info → c2cwsgiutils-6.2.0.dev75.dist-info}/entry_points.txt +0 -0
@@ -10,7 +10,6 @@ import urllib.parse
|
|
10
10
|
import uuid
|
11
11
|
import warnings
|
12
12
|
from collections.abc import Mapping
|
13
|
-
from typing import Optional, Union
|
14
13
|
|
15
14
|
import prometheus_client
|
16
15
|
import pyramid.request
|
@@ -23,7 +22,7 @@ from c2cwsgiutils import config_utils, prometheus
|
|
23
22
|
_ID_HEADERS: list[str] = []
|
24
23
|
_HTTPAdapter_send = requests.adapters.HTTPAdapter.send
|
25
24
|
_LOG = logging.getLogger(__name__)
|
26
|
-
_DEFAULT_TIMEOUT:
|
25
|
+
_DEFAULT_TIMEOUT: float | None = None
|
27
26
|
_PROMETHEUS_REQUESTS_SUMMARY = prometheus_client.Summary(
|
28
27
|
prometheus.build_metric_name("requests"),
|
29
28
|
"Requests requests",
|
@@ -35,7 +34,7 @@ _PROMETHEUS_REQUESTS_SUMMARY = prometheus_client.Summary(
|
|
35
34
|
def _gen_request_id(request: pyramid.request.Request) -> str:
|
36
35
|
for id_header in _ID_HEADERS:
|
37
36
|
if id_header in request.headers:
|
38
|
-
return request.headers[id_header] # type: ignore
|
37
|
+
return request.headers[id_header] # type: ignore[no-any-return]
|
39
38
|
return str(uuid.uuid4())
|
40
39
|
|
41
40
|
|
@@ -44,10 +43,10 @@ def _patch_requests() -> None:
|
|
44
43
|
self: requests.adapters.HTTPAdapter,
|
45
44
|
request: requests.models.PreparedRequest,
|
46
45
|
stream: bool = False,
|
47
|
-
timeout:
|
48
|
-
verify:
|
49
|
-
cert:
|
50
|
-
proxies:
|
46
|
+
timeout: None | float | tuple[float, float] | tuple[float, None] = None,
|
47
|
+
verify: bool | str = True,
|
48
|
+
cert: None | bytes | str | tuple[bytes | str, bytes | str] = None,
|
49
|
+
proxies: Mapping[str, str] | None = None,
|
51
50
|
) -> requests.Response:
|
52
51
|
pyramid_request = get_current_request()
|
53
52
|
header = _ID_HEADERS[0]
|
@@ -65,7 +64,13 @@ def _patch_requests() -> None:
|
|
65
64
|
port = parsed.port or (80 if parsed.scheme == "http" else 443)
|
66
65
|
start = time.perf_counter()
|
67
66
|
response = _HTTPAdapter_send(
|
68
|
-
self,
|
67
|
+
self,
|
68
|
+
request,
|
69
|
+
timeout=timeout,
|
70
|
+
stream=stream,
|
71
|
+
verify=verify,
|
72
|
+
cert=cert,
|
73
|
+
proxies=proxies,
|
69
74
|
)
|
70
75
|
|
71
76
|
_PROMETHEUS_REQUESTS_SUMMARY.labels(
|
@@ -81,13 +86,13 @@ def _patch_requests() -> None:
|
|
81
86
|
requests.adapters.HTTPAdapter.send = send_wrapper # type: ignore[method-assign]
|
82
87
|
|
83
88
|
|
84
|
-
def init(config:
|
89
|
+
def init(config: pyramid.config.Configurator | None = None) -> None:
|
85
90
|
"""Initialize the request tracking, for backward compatibility."""
|
86
91
|
warnings.warn("init function is deprecated; use includeme instead", stacklevel=2)
|
87
92
|
includeme(config)
|
88
93
|
|
89
94
|
|
90
|
-
def includeme(config:
|
95
|
+
def includeme(config: pyramid.config.Configurator | None = None) -> None:
|
91
96
|
"""
|
92
97
|
Initialize the request tracking.
|
93
98
|
|
@@ -103,11 +108,14 @@ def includeme(config: Optional[pyramid.config.Configurator] = None) -> None:
|
|
103
108
|
config.add_request_method(_gen_request_id, "c2c_request_id", reify=True)
|
104
109
|
|
105
110
|
_DEFAULT_TIMEOUT = config_utils.env_or_config(
|
106
|
-
config,
|
111
|
+
config,
|
112
|
+
"C2C_REQUESTS_DEFAULT_TIMEOUT",
|
113
|
+
"c2c.requests_default_timeout",
|
114
|
+
type_=float,
|
107
115
|
)
|
108
116
|
_patch_requests()
|
109
117
|
|
110
|
-
if config_utils.env_or_config(config, "C2C_SQL_REQUEST_ID", "c2c.sql_request_id", False):
|
118
|
+
if config_utils.env_or_config(config, "C2C_SQL_REQUEST_ID", "c2c.sql_request_id", default=False):
|
111
119
|
from . import _sql # pylint: disable=import-outside-toplevel
|
112
120
|
|
113
121
|
_sql.init()
|
@@ -9,7 +9,8 @@ def _add_session_id(session: Session, _transaction: Any) -> None:
|
|
9
9
|
request = get_current_request()
|
10
10
|
if request is not None:
|
11
11
|
session.execute(
|
12
|
-
sqlalchemy.text("set application_name=:session_id"),
|
12
|
+
sqlalchemy.text("set application_name=:session_id"),
|
13
|
+
params={"session_id": request.c2c_request_id},
|
13
14
|
)
|
14
15
|
|
15
16
|
|
@@ -6,14 +6,15 @@ import re
|
|
6
6
|
import subprocess # nosec
|
7
7
|
import sys
|
8
8
|
import warnings
|
9
|
-
from
|
9
|
+
from pathlib import Path
|
10
|
+
from typing import cast
|
10
11
|
|
11
12
|
_SRC_VERSION_RE = re.compile(r"^.*\(([^=]*)===?([^=]*)\)$")
|
12
13
|
_VERSION_RE = re.compile(r"^([^=]*)==([^=]*)$")
|
13
14
|
_LOG = logging.getLogger(__name__)
|
14
15
|
|
15
16
|
|
16
|
-
def _get_package_version(comp: str) -> tuple[
|
17
|
+
def _get_package_version(comp: str) -> tuple[str | None, str | None]:
|
17
18
|
"""
|
18
19
|
Parse plain and editable versions.
|
19
20
|
|
@@ -23,15 +24,14 @@ def _get_package_version(comp: str) -> tuple[Optional[str], Optional[str]]:
|
|
23
24
|
matcher = src_matcher or _VERSION_RE.match(comp)
|
24
25
|
if matcher:
|
25
26
|
return cast(tuple[str, str], matcher.groups())
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
return None, None
|
27
|
+
if len(comp) > 0 and comp[:3] != "-e ":
|
28
|
+
print("Cannot parse package version: " + comp)
|
29
|
+
return None, None
|
30
30
|
|
31
31
|
|
32
32
|
def _get_packages_version() -> dict[str, str]:
|
33
33
|
result = {}
|
34
|
-
with open(os.devnull, "w", encoding="utf-8") as devnull:
|
34
|
+
with open(os.devnull, "w", encoding="utf-8") as devnull: # noqa: PTH123
|
35
35
|
for comp in (
|
36
36
|
subprocess.check_output(["python3", "-m", "pip", "freeze"], stderr=devnull) # nosec
|
37
37
|
.decode()
|
@@ -47,7 +47,8 @@ def _get_packages_version() -> dict[str, str]:
|
|
47
47
|
def deprecated() -> None:
|
48
48
|
"""Run the command and print a deprecated notice."""
|
49
49
|
warnings.warn(
|
50
|
-
"c2cwsgiutils_genversion.py is deprecated; use c2cwsgiutils-genversion instead",
|
50
|
+
"c2cwsgiutils_genversion.py is deprecated; use c2cwsgiutils-genversion instead",
|
51
|
+
stacklevel=2,
|
51
52
|
)
|
52
53
|
return main()
|
53
54
|
|
@@ -63,7 +64,7 @@ def main() -> None:
|
|
63
64
|
report = {"main": {"git_hash": git_hash}, "packages": _get_packages_version()}
|
64
65
|
if git_tag is not None:
|
65
66
|
report["main"]["git_tag"] = git_tag
|
66
|
-
with
|
67
|
+
with Path("versions.json").open("w", encoding="utf-8") as file:
|
67
68
|
json.dump(report, file, indent=2)
|
68
69
|
|
69
70
|
|
c2cwsgiutils/scripts/stats_db.py
CHANGED
@@ -6,7 +6,6 @@ import logging
|
|
6
6
|
import os
|
7
7
|
import sys
|
8
8
|
import time
|
9
|
-
from typing import Optional
|
10
9
|
from wsgiref.simple_server import make_server
|
11
10
|
|
12
11
|
import sqlalchemy
|
@@ -30,7 +29,12 @@ def _parse_args() -> argparse.Namespace:
|
|
30
29
|
c2cwsgiutils.setup_process.fill_arguments(parser)
|
31
30
|
parser.add_argument("--db", type=str, required=True, help="DB connection string")
|
32
31
|
parser.add_argument(
|
33
|
-
"--schema",
|
32
|
+
"--schema",
|
33
|
+
type=str,
|
34
|
+
action="append",
|
35
|
+
required=True,
|
36
|
+
default=["public"],
|
37
|
+
help="schema to dump",
|
34
38
|
)
|
35
39
|
parser.add_argument(
|
36
40
|
"--extra",
|
@@ -46,7 +50,10 @@ def _parse_args() -> argparse.Namespace:
|
|
46
50
|
help="A SQL query that returns a metric name and a value, with gauge name and help",
|
47
51
|
)
|
48
52
|
parser.add_argument(
|
49
|
-
"--prometheus-url",
|
53
|
+
"--prometheus-url",
|
54
|
+
"--prometheus_url",
|
55
|
+
type=str,
|
56
|
+
help="Base URL for the Prometheus Pushgateway",
|
50
57
|
)
|
51
58
|
parser.add_argument(
|
52
59
|
"--prometheus-instance",
|
@@ -63,7 +70,7 @@ class Reporter:
|
|
63
70
|
|
64
71
|
def __init__(self, args: argparse.Namespace) -> None:
|
65
72
|
"""Initialize the reporter."""
|
66
|
-
self._error:
|
73
|
+
self._error: Exception | None = None
|
67
74
|
self.registry = CollectorRegistry()
|
68
75
|
self.prometheus_push = args.prometheus_url is not None
|
69
76
|
self.args = args
|
@@ -81,7 +88,12 @@ class Reporter:
|
|
81
88
|
return self.gauges[kind]
|
82
89
|
|
83
90
|
def do_report(
|
84
|
-
self,
|
91
|
+
self,
|
92
|
+
metric: list[str],
|
93
|
+
value: int,
|
94
|
+
kind: str,
|
95
|
+
kind_help: str,
|
96
|
+
tags: dict[str, str],
|
85
97
|
) -> None:
|
86
98
|
"""Report a metric."""
|
87
99
|
_LOG.debug("%s.%s -> %d", kind, ".".join(metric), value)
|
@@ -150,7 +162,7 @@ def _do_indexes(
|
|
150
162
|
) AS foo
|
151
163
|
ON t.tablename = foo.ctablename AND t.schemaname=foo.schemaname
|
152
164
|
WHERE t.schemaname=:schema AND t.tablename=:table
|
153
|
-
"""
|
165
|
+
""",
|
154
166
|
),
|
155
167
|
params={"schema": schema, "table": table},
|
156
168
|
):
|
@@ -185,7 +197,7 @@ def _do_table_size(
|
|
185
197
|
FROM pg_class c
|
186
198
|
LEFT JOIN pg_namespace n ON n.oid = c.relnamespace
|
187
199
|
WHERE relkind = 'r' AND nspname=:schema AND relname=:table
|
188
|
-
"""
|
200
|
+
""",
|
189
201
|
),
|
190
202
|
params={"schema": schema, "table": table},
|
191
203
|
).fetchone()
|
@@ -212,7 +224,7 @@ def _do_table_count(
|
|
212
224
|
result = session.execute(
|
213
225
|
sqlalchemy.text(
|
214
226
|
"SELECT reltuples FROM pg_class where "
|
215
|
-
"oid=(quote_ident(:schema) || '.' || quote_ident(:table))::regclass;"
|
227
|
+
"oid=(quote_ident(:schema) || '.' || quote_ident(:table))::regclass;",
|
216
228
|
),
|
217
229
|
params={"schema": schema, "table": table},
|
218
230
|
).fetchone()
|
@@ -231,7 +243,11 @@ def do_extra(session: scoped_session, sql: str, kind: str, gauge_help: str, repo
|
|
231
243
|
"""Do an extra report."""
|
232
244
|
for metric, count in session.execute(sqlalchemy.text(sql)):
|
233
245
|
reporter.do_report(
|
234
|
-
str(metric).split("."),
|
246
|
+
str(metric).split("."),
|
247
|
+
count,
|
248
|
+
kind=kind,
|
249
|
+
kind_help=gauge_help,
|
250
|
+
tags={"metric": metric},
|
235
251
|
)
|
236
252
|
|
237
253
|
|
@@ -251,7 +267,7 @@ def _do_dtats_db(args: argparse.Namespace) -> None:
|
|
251
267
|
"""
|
252
268
|
SELECT table_schema, table_name FROM information_schema.tables
|
253
269
|
WHERE table_type='BASE TABLE' AND table_schema IN :schemas
|
254
|
-
"""
|
270
|
+
""",
|
255
271
|
),
|
256
272
|
params={"schemas": tuple(args.schema)},
|
257
273
|
).fetchall()
|
@@ -259,7 +275,7 @@ def _do_dtats_db(args: argparse.Namespace) -> None:
|
|
259
275
|
_LOG.info("Process table %s.%s.", schema, table)
|
260
276
|
try:
|
261
277
|
do_table(session, schema, table, reporter)
|
262
|
-
except Exception as e: # pylint: disable=broad-
|
278
|
+
except Exception as e: # pylint: disable=broad-exception-caught
|
263
279
|
_LOG.exception("Process table %s.%s error.", schema, table)
|
264
280
|
reporter.error([schema, table], e)
|
265
281
|
|
@@ -268,7 +284,7 @@ def _do_dtats_db(args: argparse.Namespace) -> None:
|
|
268
284
|
_LOG.info("Process extra %s.", extra)
|
269
285
|
try:
|
270
286
|
do_extra(session, extra, "extra", "Extra metric", reporter)
|
271
|
-
except Exception as e: # pylint: disable=broad-
|
287
|
+
except Exception as e: # pylint: disable=broad-exception-caught
|
272
288
|
_LOG.exception("Process extra %s error.", extra)
|
273
289
|
reporter.error(["extra", str(pos + 1)], e)
|
274
290
|
if args.extra_gauge:
|
@@ -277,7 +293,7 @@ def _do_dtats_db(args: argparse.Namespace) -> None:
|
|
277
293
|
_LOG.info("Process extra %s.", extra)
|
278
294
|
try:
|
279
295
|
do_extra(session, sql, gauge, gauge_help, reporter)
|
280
|
-
except Exception as e: # pylint: disable=broad-
|
296
|
+
except Exception as e: # pylint: disable=broad-exception-caught
|
281
297
|
_LOG.exception("Process extra %s error.", extra)
|
282
298
|
reporter.error(["extra", str(len(args.extra) + pos + 1)], e)
|
283
299
|
|
c2cwsgiutils/sentry.py
CHANGED
@@ -2,8 +2,8 @@ import contextlib
|
|
2
2
|
import logging
|
3
3
|
import os
|
4
4
|
import warnings
|
5
|
-
from collections.abc import Generator, MutableMapping
|
6
|
-
from typing import Any
|
5
|
+
from collections.abc import Callable, Generator, MutableMapping
|
6
|
+
from typing import Any
|
7
7
|
|
8
8
|
import pyramid.config
|
9
9
|
import sentry_sdk.integrations
|
@@ -31,13 +31,13 @@ def _create_before_send_filter(tags: MutableMapping[str, str]) -> Callable[[Any,
|
|
31
31
|
return do_filter
|
32
32
|
|
33
33
|
|
34
|
-
def init(config:
|
34
|
+
def init(config: pyramid.config.Configurator | None = None) -> None:
|
35
35
|
"""Initialize the Sentry integration, for backward compatibility."""
|
36
36
|
warnings.warn("init function is deprecated; use includeme instead", stacklevel=2)
|
37
37
|
includeme(config)
|
38
38
|
|
39
39
|
|
40
|
-
def includeme(config:
|
40
|
+
def includeme(config: pyramid.config.Configurator | None = None) -> None:
|
41
41
|
"""Initialize the Sentry integration."""
|
42
42
|
global _CLIENT_SETUP # pylint: disable=global-statement
|
43
43
|
sentry_url = config_utils.env_or_config(config, "SENTRY_URL", "c2c.sentry.url")
|
@@ -76,45 +76,66 @@ def includeme(config: Optional[pyramid.config.Configurator] = None) -> None:
|
|
76
76
|
|
77
77
|
traces_sample_rate = float(
|
78
78
|
config_utils.env_or_config(
|
79
|
-
config,
|
80
|
-
|
79
|
+
config,
|
80
|
+
"SENTRY_TRACES_SAMPLE_RATE",
|
81
|
+
"c2c.sentry_traces_sample_rate",
|
82
|
+
"0.0",
|
83
|
+
),
|
81
84
|
)
|
82
85
|
integrations: list[sentry_sdk.integrations.Integration] = []
|
83
86
|
if config_utils.config_bool(
|
84
87
|
config_utils.env_or_config(
|
85
|
-
config,
|
86
|
-
|
88
|
+
config,
|
89
|
+
"SENTRY_INTEGRATION_LOGGING",
|
90
|
+
"c2c.sentry_integration_logging",
|
91
|
+
"true",
|
92
|
+
),
|
87
93
|
):
|
88
94
|
integrations.append(
|
89
95
|
LoggingIntegration(
|
90
96
|
level=logging.DEBUG,
|
91
97
|
event_level=config_utils.env_or_config(
|
92
|
-
config,
|
98
|
+
config,
|
99
|
+
"SENTRY_LEVEL",
|
100
|
+
"c2c.sentry_level",
|
101
|
+
"ERROR",
|
93
102
|
).upper(),
|
94
|
-
)
|
103
|
+
),
|
95
104
|
)
|
96
105
|
if config_utils.config_bool(
|
97
106
|
config_utils.env_or_config(
|
98
|
-
config,
|
99
|
-
|
107
|
+
config,
|
108
|
+
"SENTRY_INTEGRATION_PYRAMID",
|
109
|
+
"c2c.sentry_integration_pyramid",
|
110
|
+
"true",
|
111
|
+
),
|
100
112
|
):
|
101
113
|
integrations.append(PyramidIntegration())
|
102
114
|
if config_utils.config_bool(
|
103
115
|
config_utils.env_or_config(
|
104
|
-
config,
|
105
|
-
|
116
|
+
config,
|
117
|
+
"SENTRY_INTEGRATION_SQLALCHEMY",
|
118
|
+
"c2c.sentry_integration_sqlalchemy",
|
119
|
+
"true",
|
120
|
+
),
|
106
121
|
):
|
107
122
|
integrations.append(SqlalchemyIntegration())
|
108
123
|
if config_utils.config_bool(
|
109
124
|
config_utils.env_or_config(
|
110
|
-
config,
|
111
|
-
|
125
|
+
config,
|
126
|
+
"SENTRY_INTEGRATION_REDIS",
|
127
|
+
"c2c.sentry_integration_redis",
|
128
|
+
"true",
|
129
|
+
),
|
112
130
|
):
|
113
131
|
integrations.append(RedisIntegration())
|
114
132
|
if config_utils.config_bool(
|
115
133
|
config_utils.env_or_config(
|
116
|
-
config,
|
117
|
-
|
134
|
+
config,
|
135
|
+
"SENTRY_INTEGRATION_ASYNCIO",
|
136
|
+
"c2c.sentry_integration_asyncio",
|
137
|
+
"true",
|
138
|
+
),
|
118
139
|
):
|
119
140
|
integrations.append(AsyncioIntegration())
|
120
141
|
|
@@ -128,7 +149,10 @@ def includeme(config: Optional[pyramid.config.Configurator] = None) -> None:
|
|
128
149
|
_CLIENT_SETUP = True
|
129
150
|
|
130
151
|
excludes = config_utils.env_or_config(
|
131
|
-
config,
|
152
|
+
config,
|
153
|
+
"SENTRY_EXCLUDES",
|
154
|
+
"c2c.sentry.excludes",
|
155
|
+
"sentry_sdk",
|
132
156
|
).split(",")
|
133
157
|
for exclude in excludes:
|
134
158
|
ignore_logger(exclude)
|
@@ -160,7 +184,7 @@ def filter_wsgi_app(application: Callable[..., Any]) -> Callable[..., Any]:
|
|
160
184
|
try:
|
161
185
|
_LOG.info("Enable WSGI filter for Sentry")
|
162
186
|
return SentryWsgiMiddleware(application)
|
163
|
-
except Exception: # pylint: disable=broad-
|
187
|
+
except Exception: # pylint: disable=broad-exception-caught
|
164
188
|
_LOG.error("Failed enabling sentry. Continuing without it.", exc_info=True)
|
165
189
|
return application
|
166
190
|
else:
|
c2cwsgiutils/setup_process.py
CHANGED
@@ -7,7 +7,8 @@ Must be imported at the very beginning of the process's life, before any other m
|
|
7
7
|
import argparse
|
8
8
|
import logging
|
9
9
|
import warnings
|
10
|
-
from
|
10
|
+
from collections.abc import Callable
|
11
|
+
from typing import Any, TypedDict, cast
|
11
12
|
|
12
13
|
import pyramid.config
|
13
14
|
import pyramid.registry
|
@@ -57,7 +58,8 @@ def init(config_file: str = "c2c:///app/production.ini") -> None:
|
|
57
58
|
def init_logging(config_file: str = "c2c:///app/production.ini") -> None:
|
58
59
|
"""Initialize the non-WSGI application."""
|
59
60
|
warnings.warn(
|
60
|
-
"init_logging function is deprecated; use init instead so that all features are enabled",
|
61
|
+
"init_logging function is deprecated; use init instead so that all features are enabled",
|
62
|
+
stacklevel=2,
|
61
63
|
)
|
62
64
|
loader = get_config_loader(config_file)
|
63
65
|
loader.setup_logging(None)
|
@@ -82,13 +84,14 @@ def bootstrap_application_from_options(options: argparse.Namespace) -> PyramidEn
|
|
82
84
|
https://docs.pylonsproject.org/projects/pyramid/en/latest/api/paster.html?highlight=bootstrap#pyramid.paster.bootstrap
|
83
85
|
"""
|
84
86
|
return bootstrap_application(
|
85
|
-
options.config_uri,
|
87
|
+
options.config_uri,
|
88
|
+
parse_vars(options.config_vars) if options.config_vars else None,
|
86
89
|
)
|
87
90
|
|
88
91
|
|
89
92
|
def bootstrap_application(
|
90
93
|
config_uri: str = "c2c:///app/production.ini",
|
91
|
-
options:
|
94
|
+
options: dict[str, Any] | None = None,
|
92
95
|
) -> PyramidEnv:
|
93
96
|
"""
|
94
97
|
Initialize all the application.
|
@@ -51,12 +51,13 @@ class _Repository:
|
|
51
51
|
[
|
52
52
|
row[0]
|
53
53
|
for row in c.execute(
|
54
|
-
sqlalchemy.text(f"EXPLAIN ANALYZE {statement}"),
|
54
|
+
sqlalchemy.text(f"EXPLAIN ANALYZE {statement}"),
|
55
|
+
parameters,
|
55
56
|
)
|
56
|
-
]
|
57
|
+
],
|
57
58
|
)
|
58
59
|
_LOG.info(output)
|
59
|
-
except Exception: # pylint: disable=broad-
|
60
|
+
except Exception: # pylint: disable=broad-exception-caught
|
60
61
|
pass
|
61
62
|
|
62
63
|
|
@@ -75,11 +76,10 @@ def _setup_profiler(enable: str) -> None:
|
|
75
76
|
_LOG.info("Enabling the SQL profiler")
|
76
77
|
_REPOSITORY = _Repository()
|
77
78
|
sqlalchemy.event.listen(sqlalchemy.engine.Engine, "before_cursor_execute", _REPOSITORY.profile)
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
_REPOSITORY = None
|
79
|
+
elif _REPOSITORY is not None:
|
80
|
+
_LOG.info("Disabling the SQL profiler")
|
81
|
+
sqlalchemy.event.remove(sqlalchemy.engine.Engine, "before_cursor_execute", _REPOSITORY.profile)
|
82
|
+
_REPOSITORY = None
|
83
83
|
|
84
84
|
|
85
85
|
def _beautify_sql(statement: str) -> str:
|
@@ -87,8 +87,7 @@ def _beautify_sql(statement: str) -> str:
|
|
87
87
|
statement = re.sub(r" ((?:LEFT )?(?:OUTER )?JOIN )", r"\n\1", statement)
|
88
88
|
statement = re.sub(r" ON ", r"\n ON ", statement)
|
89
89
|
statement = re.sub(r" GROUP BY ", r"\nGROUP BY ", statement)
|
90
|
-
|
91
|
-
return statement
|
90
|
+
return re.sub(r" ORDER BY ", r"\nORDER BY ", statement)
|
92
91
|
|
93
92
|
|
94
93
|
def _indent(statement: str, indent: str = " ") -> str:
|
@@ -100,7 +99,9 @@ def init(config: pyramid.config.Configurator) -> None:
|
|
100
99
|
broadcast.subscribe("c2c_sql_profiler", _setup_profiler)
|
101
100
|
|
102
101
|
config.add_route(
|
103
|
-
"c2c_sql_profiler",
|
102
|
+
"c2c_sql_profiler",
|
103
|
+
config_utils.get_base_path(config) + r"/sql_profiler",
|
104
|
+
request_method="GET",
|
104
105
|
)
|
105
106
|
config.add_view(_sql_profiler_view, route_name="c2c_sql_profiler", renderer="fast_json", http_cache=0)
|
106
107
|
_LOG.info("Enabled the /sql_profiler API")
|
@@ -1,4 +1,4 @@
|
|
1
|
-
from typing import Any
|
1
|
+
from typing import Any
|
2
2
|
|
3
3
|
from sqlalchemy import Column
|
4
4
|
from sqlalchemy.orm import declarative_base
|
@@ -8,10 +8,10 @@ from sqlalchemy.types import DateTime, Integer, String
|
|
8
8
|
Base = declarative_base()
|
9
9
|
|
10
10
|
|
11
|
-
def create_log_class(tablename: str = "logs", tableargs:
|
11
|
+
def create_log_class(tablename: str = "logs", tableargs: str | dict[str, str] = "") -> Any:
|
12
12
|
"""Get the sqlalchemy lgo class."""
|
13
13
|
|
14
|
-
class Log(Base): # type: ignore
|
14
|
+
class Log(Base): # type: ignore[valid-type,misc]
|
15
15
|
"""The SQLAlchemy class that represent the log table."""
|
16
16
|
|
17
17
|
__table_args__ = tableargs
|
File without changes
|
@@ -38,10 +38,10 @@ class SQLAlchemyHandler(logging.Handler):
|
|
38
38
|
self.engine = create_engine(sqlalchemy_url["url"])
|
39
39
|
self.Log = create_log_class( # pylint: disable=invalid-name
|
40
40
|
tablename=sqlalchemy_url.get("tablename", "logs"),
|
41
|
-
tableargs=sqlalchemy_url.get("tableargs"), # type: ignore
|
41
|
+
tableargs=sqlalchemy_url.get("tableargs"), # type: ignore[arg-type]
|
42
42
|
)
|
43
43
|
Base.metadata.bind = self.engine
|
44
|
-
self.session = sessionmaker(bind=self.engine)()
|
44
|
+
self.session = sessionmaker(bind=self.engine)()
|
45
45
|
# Initialize log queue
|
46
46
|
self.log_queue: Any = queue.Queue()
|
47
47
|
# Initialize a thread to process the logs Asynchronously
|
@@ -68,10 +68,8 @@ class SQLAlchemyHandler(logging.Handler):
|
|
68
68
|
# try to reduce the number of INSERT requests to the DB
|
69
69
|
# by writing chunks of self.MAX_NB_LOGS size,
|
70
70
|
# but also do not wait forever before writing stuff (self.MAX_TIMOUT)
|
71
|
-
if (
|
72
|
-
|
73
|
-
and (len(logs) >= self.MAX_NB_LOGS)
|
74
|
-
or (time.perf_counter() >= (time_since_last + self.MAX_TIMEOUT))
|
71
|
+
if (logs and (len(logs) >= self.MAX_NB_LOGS)) or (
|
72
|
+
time.perf_counter() >= (time_since_last + self.MAX_TIMEOUT)
|
75
73
|
):
|
76
74
|
self._write_logs(logs)
|
77
75
|
break
|
@@ -87,7 +85,7 @@ class SQLAlchemyHandler(logging.Handler):
|
|
87
85
|
self.session.rollback()
|
88
86
|
self.session.bulk_save_objects(logs)
|
89
87
|
self.session.commit()
|
90
|
-
except Exception as e: # pylint: disable=broad-
|
88
|
+
except Exception as e: # pylint: disable=broad-exception-caught #
|
91
89
|
# if we really cannot commit the log to DB, do not lock the
|
92
90
|
# thread and do not crash the application
|
93
91
|
_LOG.critical(e)
|
@@ -101,8 +99,9 @@ class SQLAlchemyHandler(logging.Handler):
|
|
101
99
|
create_database(self.engine.url)
|
102
100
|
# FIXME: we should not access directly the private __table_args__ # pylint: disable=fixme
|
103
101
|
# variable, but add an accessor method in models.Log class
|
104
|
-
if
|
105
|
-
"schema",
|
102
|
+
if self.Log.__table_args__ is not None and self.Log.__table_args__.get(
|
103
|
+
"schema",
|
104
|
+
None,
|
106
105
|
):
|
107
106
|
with self.engine.begin() as connection:
|
108
107
|
if not self.engine.dialect.has_schema(connection, self.Log.__table_args__["schema"]):
|
@@ -1,7 +1,8 @@
|
|
1
1
|
import logging
|
2
2
|
import re
|
3
3
|
import time
|
4
|
-
from
|
4
|
+
from collections.abc import Callable
|
5
|
+
from typing import Any
|
5
6
|
|
6
7
|
import prometheus_client
|
7
8
|
import sqlalchemy.event
|
@@ -77,10 +78,18 @@ def _create_sqlalchemy_timer_cb(what: str) -> Callable[..., Any]:
|
|
77
78
|
|
78
79
|
|
79
80
|
def _before_cursor_execute(
|
80
|
-
conn: Connection,
|
81
|
+
conn: Connection,
|
82
|
+
_cursor: Any,
|
83
|
+
statement: str,
|
84
|
+
_parameters: Any,
|
85
|
+
_context: Any,
|
86
|
+
_executemany: Any,
|
81
87
|
) -> None:
|
82
88
|
sqlalchemy.event.listen(
|
83
|
-
conn,
|
89
|
+
conn,
|
90
|
+
"after_cursor_execute",
|
91
|
+
_create_sqlalchemy_timer_cb(_simplify_sql(statement)),
|
92
|
+
once=True,
|
84
93
|
)
|
85
94
|
|
86
95
|
|
@@ -1,6 +1,6 @@
|
|
1
1
|
import logging
|
2
2
|
import time
|
3
|
-
from
|
3
|
+
from collections.abc import Callable
|
4
4
|
|
5
5
|
import prometheus_client
|
6
6
|
import pyramid.config
|
@@ -29,8 +29,8 @@ _PROMETHEUS_PYRAMID_VIEWS_SUMMARY = prometheus_client.Summary(
|
|
29
29
|
def _add_server_metric(
|
30
30
|
request: pyramid.request.Request,
|
31
31
|
name: str,
|
32
|
-
duration:
|
33
|
-
description:
|
32
|
+
duration: float | None = None,
|
33
|
+
description: str | None = None,
|
34
34
|
) -> None:
|
35
35
|
# format: <name>;due=<duration>;desc=<description>
|
36
36
|
metric = name
|
@@ -46,16 +46,14 @@ def _add_server_metric(
|
|
46
46
|
|
47
47
|
|
48
48
|
def _create_finished_cb(
|
49
|
-
kind: str,
|
49
|
+
kind: str,
|
50
|
+
measure: prometheus_client.Summary,
|
50
51
|
) -> Callable[[pyramid.request.Request], None]: # pragma: nocover
|
51
52
|
start = time.process_time()
|
52
53
|
|
53
54
|
def finished_cb(request: pyramid.request.Request) -> None:
|
54
55
|
if request.exception is not None:
|
55
|
-
if isinstance(request.exception, HTTPException)
|
56
|
-
status = request.exception.code
|
57
|
-
else:
|
58
|
-
status = 599
|
56
|
+
status = request.exception.code if isinstance(request.exception, HTTPException) else 599
|
59
57
|
else:
|
60
58
|
status = request.response.status_code
|
61
59
|
if request.matched_route is None:
|
@@ -74,7 +72,10 @@ def _create_finished_cb(
|
|
74
72
|
status,
|
75
73
|
)
|
76
74
|
measure.labels(
|
77
|
-
method=request.method,
|
75
|
+
method=request.method,
|
76
|
+
route=name,
|
77
|
+
status=status,
|
78
|
+
group=str(status // 100 * 100),
|
78
79
|
).observe(time.process_time() - start)
|
79
80
|
_add_server_metric(request, kind, duration=time.process_time() - start)
|
80
81
|
|