python-json-logger 4.1.0__tar.gz → 4.3.0.dev2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- python_json_logger-4.3.0.dev2/MANIFEST.in +5 -0
- {python_json_logger-4.1.0/src/python_json_logger.egg-info → python_json_logger-4.3.0.dev2}/PKG-INFO +2 -20
- {python_json_logger-4.1.0 → python_json_logger-4.3.0.dev2}/pyproject.toml +17 -4
- python_json_logger-4.3.0.dev2/scripts/benchmark_plots.py +234 -0
- python_json_logger-4.3.0.dev2/scripts/benchmark_report.py +301 -0
- {python_json_logger-4.1.0 → python_json_logger-4.3.0.dev2/src/python_json_logger.egg-info}/PKG-INFO +2 -20
- {python_json_logger-4.1.0 → python_json_logger-4.3.0.dev2}/src/python_json_logger.egg-info/SOURCES.txt +8 -3
- {python_json_logger-4.1.0 → python_json_logger-4.3.0.dev2}/src/pythonjsonlogger/core.py +58 -28
- {python_json_logger-4.1.0 → python_json_logger-4.3.0.dev2}/src/pythonjsonlogger/defaults.py +9 -1
- python_json_logger-4.3.0.dev2/tests/test_benchmark_report.py +241 -0
- python_json_logger-4.3.0.dev2/tests/test_cookbook.py +58 -0
- {python_json_logger-4.1.0 → python_json_logger-4.3.0.dev2}/tests/test_dictconfig.py +30 -0
- {python_json_logger-4.1.0 → python_json_logger-4.3.0.dev2}/tests/test_formatters.py +464 -5
- python_json_logger-4.3.0.dev2/tests/test_publish_docs.py +31 -0
- python_json_logger-4.3.0.dev2/tests/test_quickstart.py +39 -0
- python_json_logger-4.3.0.dev2/tests/test_release.py +184 -0
- python_json_logger-4.1.0/MANIFEST.in +0 -3
- python_json_logger-4.1.0/NOTICE +0 -5
- python_json_logger-4.1.0/src/python_json_logger.egg-info/requires.txt +0 -22
- {python_json_logger-4.1.0 → python_json_logger-4.3.0.dev2}/LICENSE +0 -0
- {python_json_logger-4.1.0 → python_json_logger-4.3.0.dev2}/README.md +0 -0
- {python_json_logger-4.1.0 → python_json_logger-4.3.0.dev2}/setup.cfg +0 -0
- {python_json_logger-4.1.0 → python_json_logger-4.3.0.dev2}/src/python_json_logger.egg-info/dependency_links.txt +0 -0
- {python_json_logger-4.1.0 → python_json_logger-4.3.0.dev2}/src/python_json_logger.egg-info/top_level.txt +0 -0
- {python_json_logger-4.1.0 → python_json_logger-4.3.0.dev2}/src/pythonjsonlogger/__init__.py +0 -0
- {python_json_logger-4.1.0 → python_json_logger-4.3.0.dev2}/src/pythonjsonlogger/exception.py +0 -0
- {python_json_logger-4.1.0 → python_json_logger-4.3.0.dev2}/src/pythonjsonlogger/json.py +0 -0
- {python_json_logger-4.1.0 → python_json_logger-4.3.0.dev2}/src/pythonjsonlogger/jsonlogger.py +0 -0
- {python_json_logger-4.1.0 → python_json_logger-4.3.0.dev2}/src/pythonjsonlogger/msgspec.py +0 -0
- {python_json_logger-4.1.0 → python_json_logger-4.3.0.dev2}/src/pythonjsonlogger/orjson.py +0 -0
- {python_json_logger-4.1.0 → python_json_logger-4.3.0.dev2}/src/pythonjsonlogger/py.typed +0 -0
- {python_json_logger-4.1.0 → python_json_logger-4.3.0.dev2}/src/pythonjsonlogger/utils.py +0 -0
- {python_json_logger-4.1.0 → python_json_logger-4.3.0.dev2}/tests/__init__.py +0 -0
- {python_json_logger-4.1.0 → python_json_logger-4.3.0.dev2}/tests/test_deprecation.py +0 -0
- {python_json_logger-4.1.0 → python_json_logger-4.3.0.dev2}/tests/test_missing.py +0 -0
{python_json_logger-4.1.0/src/python_json_logger.egg-info → python_json_logger-4.3.0.dev2}/PKG-INFO
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: python-json-logger
|
|
3
|
-
Version: 4.
|
|
3
|
+
Version: 4.3.0.dev2
|
|
4
4
|
Summary: JSON Log Formatter for the Python Logging Package
|
|
5
5
|
Author-email: Zakaria Zajac <zak@madzak.com>, Nicholas Hairs <info+python-json-logger@nicholashairs.com>
|
|
6
6
|
Maintainer-email: Nicholas Hairs <info+python-json-logger@nicholashairs.com>
|
|
@@ -16,30 +16,12 @@ Classifier: Programming Language :: Python :: 3.11
|
|
|
16
16
|
Classifier: Programming Language :: Python :: 3.12
|
|
17
17
|
Classifier: Programming Language :: Python :: 3.13
|
|
18
18
|
Classifier: Programming Language :: Python :: 3.14
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.15
|
|
19
20
|
Classifier: Topic :: System :: Logging
|
|
20
21
|
Classifier: Typing :: Typed
|
|
21
22
|
Requires-Python: >=3.10
|
|
22
23
|
Description-Content-Type: text/markdown
|
|
23
24
|
License-File: LICENSE
|
|
24
|
-
Provides-Extra: dev
|
|
25
|
-
Requires-Dist: orjson; implementation_name != "pypy" and extra == "dev"
|
|
26
|
-
Requires-Dist: msgspec; implementation_name != "pypy" and extra == "dev"
|
|
27
|
-
Requires-Dist: validate-pyproject[all]; extra == "dev"
|
|
28
|
-
Requires-Dist: black; extra == "dev"
|
|
29
|
-
Requires-Dist: pylint; extra == "dev"
|
|
30
|
-
Requires-Dist: mypy; extra == "dev"
|
|
31
|
-
Requires-Dist: pytest; extra == "dev"
|
|
32
|
-
Requires-Dist: freezegun; extra == "dev"
|
|
33
|
-
Requires-Dist: tzdata; extra == "dev"
|
|
34
|
-
Requires-Dist: build; extra == "dev"
|
|
35
|
-
Requires-Dist: mkdocs; extra == "dev"
|
|
36
|
-
Requires-Dist: mkdocs-material>=8.5; extra == "dev"
|
|
37
|
-
Requires-Dist: mkdocs-awesome-pages-plugin; extra == "dev"
|
|
38
|
-
Requires-Dist: mdx_truly_sane_lists; extra == "dev"
|
|
39
|
-
Requires-Dist: mkdocstrings[python]; extra == "dev"
|
|
40
|
-
Requires-Dist: mkdocs-gen-files; extra == "dev"
|
|
41
|
-
Requires-Dist: mkdocs-literate-nav; extra == "dev"
|
|
42
|
-
Requires-Dist: mike; extra == "dev"
|
|
43
25
|
Dynamic: license-file
|
|
44
26
|
|
|
45
27
|
[](https://pypi.python.org/pypi/python-json-logger/)
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "python-json-logger"
|
|
7
|
-
version = "4.
|
|
7
|
+
version = "4.3.0.dev2"
|
|
8
8
|
description = "JSON Log Formatter for the Python Logging Package"
|
|
9
9
|
authors = [
|
|
10
10
|
{name = "Zakaria Zajac", email = "zak@madzak.com"},
|
|
@@ -31,6 +31,7 @@ classifiers = [
|
|
|
31
31
|
"Programming Language :: Python :: 3.12",
|
|
32
32
|
"Programming Language :: Python :: 3.13",
|
|
33
33
|
"Programming Language :: Python :: 3.14",
|
|
34
|
+
"Programming Language :: Python :: 3.15",
|
|
34
35
|
"Topic :: System :: Logging",
|
|
35
36
|
"Typing :: Typed",
|
|
36
37
|
]
|
|
@@ -39,7 +40,15 @@ classifiers = [
|
|
|
39
40
|
Homepage = "https://nhairs.github.io/python-json-logger"
|
|
40
41
|
GitHub = "https://github.com/nhairs/python-json-logger"
|
|
41
42
|
|
|
42
|
-
[
|
|
43
|
+
[dependency-groups]
|
|
44
|
+
benchmark-report = ["matplotlib>=3.8,<4"]
|
|
45
|
+
benchmark = [
|
|
46
|
+
# pytest-benchmark 5.x uses Node._nodeid, removed in pytest 9.2.
|
|
47
|
+
"pytest<9.2",
|
|
48
|
+
"pytest-benchmark>=5.1,<6",
|
|
49
|
+
"orjson;implementation_name!='pypy'",
|
|
50
|
+
"msgspec;implementation_name!='pypy'",
|
|
51
|
+
]
|
|
43
52
|
dev = [
|
|
44
53
|
## Optional but required for dev
|
|
45
54
|
"orjson;implementation_name!='pypy'",
|
|
@@ -48,7 +57,7 @@ dev = [
|
|
|
48
57
|
"validate-pyproject[all]",
|
|
49
58
|
"black",
|
|
50
59
|
"pylint",
|
|
51
|
-
"
|
|
60
|
+
"pyrefly",
|
|
52
61
|
## Test
|
|
53
62
|
"pytest",
|
|
54
63
|
"freezegun",
|
|
@@ -56,7 +65,7 @@ dev = [
|
|
|
56
65
|
## Build
|
|
57
66
|
"build",
|
|
58
67
|
## Docs
|
|
59
|
-
"mkdocs",
|
|
68
|
+
"mkdocs<2",
|
|
60
69
|
"mkdocs-material>=8.5",
|
|
61
70
|
"mkdocs-awesome-pages-plugin",
|
|
62
71
|
"mdx_truly_sane_lists",
|
|
@@ -64,6 +73,7 @@ dev = [
|
|
|
64
73
|
"mkdocs-gen-files",
|
|
65
74
|
"mkdocs-literate-nav",
|
|
66
75
|
"mike",
|
|
76
|
+
"packaging",
|
|
67
77
|
]
|
|
68
78
|
|
|
69
79
|
[tool.setuptools.packages.find]
|
|
@@ -75,3 +85,6 @@ pythonjsonlogger = ["py.typed"]
|
|
|
75
85
|
|
|
76
86
|
[tool.black]
|
|
77
87
|
line-length = 100
|
|
88
|
+
|
|
89
|
+
[tool.pytest.ini_options]
|
|
90
|
+
testpaths = ["tests"]
|
|
@@ -0,0 +1,234 @@
|
|
|
1
|
+
"""Export timing and speedup plots from the committed benchmark summaries."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
import matplotlib
|
|
9
|
+
|
|
10
|
+
matplotlib.use("Agg")
|
|
11
|
+
import matplotlib.pyplot as plt
|
|
12
|
+
from matplotlib.figure import Figure
|
|
13
|
+
|
|
14
|
+
from benchmark_report import (
|
|
15
|
+
LAYERS,
|
|
16
|
+
WORKLOADS,
|
|
17
|
+
comparison_groups,
|
|
18
|
+
formatting_summary,
|
|
19
|
+
load_results,
|
|
20
|
+
render,
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
COLORS = {
|
|
24
|
+
"json (CPython)": "#0072B2",
|
|
25
|
+
"orjson (CPython)": "#D55E00",
|
|
26
|
+
"msgspec (CPython)": "#009E73",
|
|
27
|
+
"json (PyPy)": "#CC79A7",
|
|
28
|
+
}
|
|
29
|
+
STYLE = {
|
|
30
|
+
"font.family": "DejaVu Sans",
|
|
31
|
+
"svg.fonttype": "none",
|
|
32
|
+
"svg.hashsalt": "python-json-logger-benchmark",
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def make_speedup_plot(result: dict) -> Figure:
|
|
37
|
+
"""Compare grouped speedup bars in two panels with workload boundaries."""
|
|
38
|
+
backends = result["backends"]
|
|
39
|
+
cases = result["cases"]
|
|
40
|
+
reference = "json (CPython)" if "json (CPython)" in backends else "json (PyPy)"
|
|
41
|
+
figure, axes = plt.subplots(1, 2, figsize=(13, 8))
|
|
42
|
+
height = 0.8 / len(backends)
|
|
43
|
+
for axis, layer in zip(axes, LAYERS):
|
|
44
|
+
upper = 1.0
|
|
45
|
+
for index, backend in enumerate(backends):
|
|
46
|
+
positions = [row - 0.4 + height * (index + 0.5) for row in range(len(WORKLOADS))]
|
|
47
|
+
values = [
|
|
48
|
+
cases[layer, workload, reference]["median"]
|
|
49
|
+
/ cases[layer, workload, backend]["median"]
|
|
50
|
+
for workload in WORKLOADS
|
|
51
|
+
]
|
|
52
|
+
upper = max(upper, max(values))
|
|
53
|
+
axis.barh(
|
|
54
|
+
positions,
|
|
55
|
+
values,
|
|
56
|
+
height=height * 0.85,
|
|
57
|
+
color=COLORS[backend],
|
|
58
|
+
label=backend,
|
|
59
|
+
zorder=2,
|
|
60
|
+
)
|
|
61
|
+
for position, value in zip(positions, values):
|
|
62
|
+
axis.annotate(
|
|
63
|
+
f"{value:.2g}×",
|
|
64
|
+
(value, position),
|
|
65
|
+
xytext=(4, 0),
|
|
66
|
+
textcoords="offset points",
|
|
67
|
+
va="center",
|
|
68
|
+
fontsize=8,
|
|
69
|
+
)
|
|
70
|
+
axis.set_yticks(range(len(WORKLOADS)), WORKLOADS)
|
|
71
|
+
axis.set_ylim(len(WORKLOADS) - 0.5, -0.5)
|
|
72
|
+
axis.set_xlim(0, upper * 1.2)
|
|
73
|
+
for row in range(len(WORKLOADS)):
|
|
74
|
+
if row % 2 == 0:
|
|
75
|
+
axis.axhspan(row - 0.5, row + 0.5, color="#000000", alpha=0.045, zorder=0)
|
|
76
|
+
axis.axhline(row + 0.5, color="#888888", alpha=0.3, linewidth=0.7)
|
|
77
|
+
axis.set_title("Serialization" if layer == "serialization" else "Complete formatting")
|
|
78
|
+
axis.set_xlabel("Speedup (×) — higher is faster")
|
|
79
|
+
axis.grid(axis="x", alpha=0.25)
|
|
80
|
+
axis.set_axisbelow(True)
|
|
81
|
+
figure.suptitle(f"Python {result['python']}: Ratios of median times", fontsize=15)
|
|
82
|
+
handles, labels = axes[0].get_legend_handles_labels()
|
|
83
|
+
figure.legend(
|
|
84
|
+
handles, labels, loc="lower center", bbox_to_anchor=(0.5, 0.065), ncol=len(backends)
|
|
85
|
+
)
|
|
86
|
+
figure.text(
|
|
87
|
+
0.5,
|
|
88
|
+
0.025,
|
|
89
|
+
f"Speedup relative to {reference} (1×). Descriptive ratios have no confidence intervals.",
|
|
90
|
+
ha="center",
|
|
91
|
+
fontsize=9,
|
|
92
|
+
)
|
|
93
|
+
figure.tight_layout(rect=(0, 0.14, 1, 0.94))
|
|
94
|
+
return figure
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def make_plot(result: dict, kind: str) -> Figure:
|
|
98
|
+
"""Compare labeled horizontal bars in separate linear-scale workload panels."""
|
|
99
|
+
if kind == "speedup":
|
|
100
|
+
return make_speedup_plot(result)
|
|
101
|
+
backends = result["backends"]
|
|
102
|
+
cases = result["cases"]
|
|
103
|
+
figure, axes = plt.subplots(len(WORKLOADS), len(LAYERS), figsize=(13, 16))
|
|
104
|
+
for row, workload in enumerate(WORKLOADS):
|
|
105
|
+
for col, layer in enumerate(LAYERS):
|
|
106
|
+
axis = axes[row, col]
|
|
107
|
+
stats = [cases[layer, workload, backend] for backend in backends]
|
|
108
|
+
values = [item["median"] * 1e6 for item in stats]
|
|
109
|
+
labels = [f"{value:.3g}" for value in values]
|
|
110
|
+
axis.barh(
|
|
111
|
+
range(len(backends)),
|
|
112
|
+
values,
|
|
113
|
+
height=0.65,
|
|
114
|
+
color=[COLORS[backend] for backend in backends],
|
|
115
|
+
alpha=0.85,
|
|
116
|
+
zorder=2,
|
|
117
|
+
)
|
|
118
|
+
upper = max(values)
|
|
119
|
+
for index, label in enumerate(labels):
|
|
120
|
+
axis.text(values[index] + upper * 0.025, index, label, va="center", fontsize=9)
|
|
121
|
+
axis.set_yticks(range(len(backends)), backends, fontsize=9)
|
|
122
|
+
axis.invert_yaxis()
|
|
123
|
+
axis.set_xlim(0, upper * 1.23)
|
|
124
|
+
layer_title = "Serialization" if layer == "serialization" else "Complete formatting"
|
|
125
|
+
axis.set_title(f"{workload} · {layer_title}", fontsize=11)
|
|
126
|
+
axis.set_xlabel("Median time (µs)", fontsize=9)
|
|
127
|
+
axis.grid(axis="x", alpha=0.25, zorder=0)
|
|
128
|
+
axis.set_axisbelow(True)
|
|
129
|
+
axis.spines[["top", "right"]].set_visible(False)
|
|
130
|
+
figure.suptitle(f"Python {result['python']}: Median operation times", fontsize=15)
|
|
131
|
+
caption = (
|
|
132
|
+
"Every panel starts at zero; axis ranges differ by panel. Lower is faster.\n"
|
|
133
|
+
"Labels show median µs. Variation statistics are available in the tables."
|
|
134
|
+
)
|
|
135
|
+
figure.text(0.5, 0.012, caption, ha="center", fontsize=10)
|
|
136
|
+
figure.tight_layout(rect=(0, 0.045, 1, 0.97), h_pad=1.5, w_pad=2)
|
|
137
|
+
return figure
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def make_summary_plot(summary: dict) -> Figure:
|
|
141
|
+
"""Combine workload timing and speedup panels into one shareable figure."""
|
|
142
|
+
figure, axes = plt.subplots(len(WORKLOADS), 2, figsize=(13, 16))
|
|
143
|
+
for row, workload in enumerate(WORKLOADS):
|
|
144
|
+
for col, kind in enumerate(("times", "speedup")):
|
|
145
|
+
axis = axes[row, col]
|
|
146
|
+
backends = [
|
|
147
|
+
backend for backend, metrics in summary[workload].items() if kind in metrics
|
|
148
|
+
]
|
|
149
|
+
stats = [summary[workload][backend][kind] for backend in backends]
|
|
150
|
+
values = [item["mean"] for item in stats]
|
|
151
|
+
errors = [
|
|
152
|
+
[item["mean"] - item["min"] for item in stats],
|
|
153
|
+
[item["max"] - item["mean"] for item in stats],
|
|
154
|
+
]
|
|
155
|
+
axis.barh(
|
|
156
|
+
range(len(backends)),
|
|
157
|
+
values,
|
|
158
|
+
xerr=errors,
|
|
159
|
+
capsize=4,
|
|
160
|
+
color=[COLORS[backend] for backend in backends],
|
|
161
|
+
alpha=0.85,
|
|
162
|
+
error_kw={"elinewidth": 1.2},
|
|
163
|
+
zorder=2,
|
|
164
|
+
)
|
|
165
|
+
upper = max((item["max"] for item in stats), default=1)
|
|
166
|
+
for index, item in enumerate(stats):
|
|
167
|
+
precision = 3 if kind == "times" else 2
|
|
168
|
+
suffix = "" if kind == "times" else "×"
|
|
169
|
+
axis.text(
|
|
170
|
+
item["max"] + upper * 0.025,
|
|
171
|
+
index,
|
|
172
|
+
f"{item['mean']:.{precision}g}{suffix}",
|
|
173
|
+
va="center",
|
|
174
|
+
fontsize=9,
|
|
175
|
+
)
|
|
176
|
+
axis.set_yticks(range(len(backends)), backends, fontsize=9)
|
|
177
|
+
axis.invert_yaxis()
|
|
178
|
+
axis.set_xlim(0, upper * 1.25)
|
|
179
|
+
axis.set_title(f"{workload} · {'Timing' if kind == 'times' else 'Speedup'}")
|
|
180
|
+
axis.set_xlabel(
|
|
181
|
+
"Time (µs) — lower is faster"
|
|
182
|
+
if kind == "times"
|
|
183
|
+
else "Speedup (×) — higher is faster"
|
|
184
|
+
)
|
|
185
|
+
if kind == "speedup":
|
|
186
|
+
axis.axvline(1, color="#555555", linestyle="--", linewidth=1)
|
|
187
|
+
axis.grid(axis="x", alpha=0.25)
|
|
188
|
+
axis.set_axisbelow(True)
|
|
189
|
+
axis.spines[["top", "right"]].set_visible(False)
|
|
190
|
+
figure.suptitle("Complete formatting across Python versions", fontsize=15)
|
|
191
|
+
figure.text(
|
|
192
|
+
0.5,
|
|
193
|
+
0.012,
|
|
194
|
+
"Bars: arithmetic mean across versions. Whiskers: version minimum–maximum, not sample variation.\n"
|
|
195
|
+
"Speedups use version-matched CPython stdlib JSON. Scales and version coverage differ; see the documentation.",
|
|
196
|
+
ha="center",
|
|
197
|
+
fontsize=10,
|
|
198
|
+
)
|
|
199
|
+
figure.tight_layout(rect=(0, 0.045, 1, 0.97), h_pad=1.5, w_pad=2)
|
|
200
|
+
return figure
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def save_plot(figure: Figure, target: Path) -> None:
|
|
204
|
+
"""Save SVG without timestamps or trailing whitespace."""
|
|
205
|
+
figure.savefig(target, metadata={"Date": None})
|
|
206
|
+
target.write_text(
|
|
207
|
+
"\n".join(line.rstrip() for line in target.read_text(encoding="utf-8").splitlines()) + "\n",
|
|
208
|
+
encoding="utf-8",
|
|
209
|
+
)
|
|
210
|
+
plt.close(figure)
|
|
211
|
+
return
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def main() -> None:
|
|
215
|
+
"""Write deterministic SVG files; install the benchmark-report dependency group."""
|
|
216
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
217
|
+
parser.add_argument("results", type=Path, nargs="+")
|
|
218
|
+
parser.add_argument("--output", type=Path, required=True)
|
|
219
|
+
args = parser.parse_args()
|
|
220
|
+
render(args.results) # Validate matrices and statistics before producing any plots.
|
|
221
|
+
args.output.mkdir(parents=True, exist_ok=True)
|
|
222
|
+
with matplotlib.rc_context(STYLE):
|
|
223
|
+
for result in comparison_groups(load_results(args.results)):
|
|
224
|
+
for kind in ("times", "speedup"):
|
|
225
|
+
figure = make_plot(result, kind)
|
|
226
|
+
target = args.output / f"python-{result['python']}-{kind}.svg"
|
|
227
|
+
save_plot(figure, target)
|
|
228
|
+
summary = formatting_summary(comparison_groups(load_results(args.results)))
|
|
229
|
+
save_plot(make_summary_plot(summary), args.output / "summary.svg")
|
|
230
|
+
return
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
if __name__ == "__main__":
|
|
234
|
+
main()
|
|
@@ -0,0 +1,301 @@
|
|
|
1
|
+
"""Generate benchmark tables from pytest-benchmark JSON without extra dependencies."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import hashlib
|
|
7
|
+
import json
|
|
8
|
+
import math
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from statistics import fmean
|
|
11
|
+
from textwrap import indent
|
|
12
|
+
|
|
13
|
+
START = "<!-- benchmark-results:start -->"
|
|
14
|
+
END = "<!-- benchmark-results:end -->"
|
|
15
|
+
SUMMARY_START = "<!-- benchmark-summary:start -->"
|
|
16
|
+
SUMMARY_END = "<!-- benchmark-summary:end -->"
|
|
17
|
+
SERIES = ("json (CPython)", "orjson (CPython)", "msgspec (CPython)", "json (PyPy)")
|
|
18
|
+
BACKENDS = ("json", "orjson", "msgspec")
|
|
19
|
+
LAYERS = ("serialization", "formatting")
|
|
20
|
+
WORKLOADS = ("simple", "structured", "nested", "unicode", "rich", "exception")
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def load_results(paths: list[Path]) -> list[dict]:
|
|
24
|
+
"""Load summary statistics in deterministic runtime order."""
|
|
25
|
+
results = []
|
|
26
|
+
for path in paths:
|
|
27
|
+
result = json.loads(path.read_text(encoding="utf-8"))
|
|
28
|
+
# Raw exports can contain millions of samples; retain only table inputs.
|
|
29
|
+
for benchmark in result["benchmarks"]:
|
|
30
|
+
benchmark["stats"].pop("data", None)
|
|
31
|
+
results.append(result)
|
|
32
|
+
results.sort(
|
|
33
|
+
key=lambda result: (
|
|
34
|
+
tuple(map(int, result["environment"]["python"].split(".")[:2])),
|
|
35
|
+
result["environment"].get("implementation", "CPython"),
|
|
36
|
+
)
|
|
37
|
+
)
|
|
38
|
+
return results
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def comparison_groups(results: list[dict]) -> list[dict]:
|
|
42
|
+
"""Validate exports and group implementations by Python major/minor version."""
|
|
43
|
+
groups: dict[str, dict] = {}
|
|
44
|
+
runtimes = set()
|
|
45
|
+
for result in results:
|
|
46
|
+
environment = result["environment"]
|
|
47
|
+
python = environment["python"]
|
|
48
|
+
implementation = environment.get("implementation", "CPython")
|
|
49
|
+
minor = ".".join(python.split(".")[:2])
|
|
50
|
+
runtime = (implementation, minor)
|
|
51
|
+
if implementation not in ("CPython", "PyPy"):
|
|
52
|
+
raise ValueError(f"Unsupported implementation: {implementation}")
|
|
53
|
+
if runtime in runtimes:
|
|
54
|
+
raise ValueError(f"Duplicate runtime: {runtime}")
|
|
55
|
+
runtimes.add(runtime)
|
|
56
|
+
backends = tuple(environment.get("backends", BACKENDS))
|
|
57
|
+
if "json" not in backends or not set(backends).issubset(BACKENDS):
|
|
58
|
+
raise ValueError(f"Invalid backend list for Python {python}")
|
|
59
|
+
cases = {}
|
|
60
|
+
for benchmark in result["benchmarks"]:
|
|
61
|
+
info = benchmark["extra_info"]
|
|
62
|
+
key = (info["layer"], info["workload"], info["backend"])
|
|
63
|
+
if key in cases:
|
|
64
|
+
raise ValueError(f"Duplicate case: {key}")
|
|
65
|
+
median = benchmark["stats"]["median"]
|
|
66
|
+
if not math.isfinite(median) or median <= 0:
|
|
67
|
+
raise ValueError(f"Invalid median: {key}")
|
|
68
|
+
stats = benchmark["stats"]
|
|
69
|
+
for name in ("mean", "stddev", "iqr", "q1", "q3"):
|
|
70
|
+
if not math.isfinite(stats[name]) or stats[name] < 0:
|
|
71
|
+
raise ValueError(f"Invalid {name}: {key}")
|
|
72
|
+
if stats["mean"] <= 0:
|
|
73
|
+
raise ValueError(f"Invalid mean: {key}")
|
|
74
|
+
if not 0 < stats["q1"] <= median <= stats["q3"]:
|
|
75
|
+
raise ValueError(f"Invalid quartiles: {key}")
|
|
76
|
+
if stats["rounds"] < 1 or stats["iterations"] < 1:
|
|
77
|
+
raise ValueError(f"Invalid repetition counts: {key}")
|
|
78
|
+
cases[key] = stats
|
|
79
|
+
required = {
|
|
80
|
+
(layer, workload, backend)
|
|
81
|
+
for layer in LAYERS
|
|
82
|
+
for workload in WORKLOADS
|
|
83
|
+
for backend in backends
|
|
84
|
+
}
|
|
85
|
+
if set(cases) != required:
|
|
86
|
+
raise ValueError(f"Incomplete or unexpected matrix for Python {python}")
|
|
87
|
+
group = groups.setdefault(
|
|
88
|
+
minor, {"python": minor, "sources": [], "cases": {}, "backends": []}
|
|
89
|
+
)
|
|
90
|
+
group["sources"].append(result)
|
|
91
|
+
for backend in backends:
|
|
92
|
+
label = f"{backend} ({implementation})"
|
|
93
|
+
group["backends"].append(label)
|
|
94
|
+
for layer in LAYERS:
|
|
95
|
+
for workload in WORKLOADS:
|
|
96
|
+
group["cases"][layer, workload, label] = cases[layer, workload, backend]
|
|
97
|
+
return list(groups.values())
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def formatting_summary(groups: list[dict]) -> dict:
|
|
101
|
+
"""Aggregate per-version medians and matched CPython-relative ratios.
|
|
102
|
+
|
|
103
|
+
Missing CPython references contribute timings but no speedup ratios.
|
|
104
|
+
Each metric retains its own version coverage.
|
|
105
|
+
"""
|
|
106
|
+
summary = {}
|
|
107
|
+
for workload in WORKLOADS:
|
|
108
|
+
summary[workload] = {}
|
|
109
|
+
for backend in SERIES:
|
|
110
|
+
metrics = {}
|
|
111
|
+
for kind in ("times", "speedup"):
|
|
112
|
+
values = []
|
|
113
|
+
versions = []
|
|
114
|
+
for group in groups:
|
|
115
|
+
cases = group["cases"]
|
|
116
|
+
key = ("formatting", workload, backend)
|
|
117
|
+
reference = ("formatting", workload, "json (CPython)")
|
|
118
|
+
if key not in cases or (kind == "speedup" and reference not in cases):
|
|
119
|
+
continue
|
|
120
|
+
timing = cases[key]["median"]
|
|
121
|
+
values.append(
|
|
122
|
+
timing * 1e6 if kind == "times" else cases[reference]["median"] / timing
|
|
123
|
+
)
|
|
124
|
+
versions.append(group["python"])
|
|
125
|
+
if values:
|
|
126
|
+
metrics[kind] = {
|
|
127
|
+
"mean": fmean(values),
|
|
128
|
+
"min": min(values),
|
|
129
|
+
"max": max(values),
|
|
130
|
+
"versions": versions,
|
|
131
|
+
}
|
|
132
|
+
if metrics:
|
|
133
|
+
summary[workload][backend] = metrics
|
|
134
|
+
return summary
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def render(paths: list[Path], plots: str | None = None) -> str:
|
|
138
|
+
"""Render version sections comparing encoders and Python implementations."""
|
|
139
|
+
results = load_results(paths)
|
|
140
|
+
sections = []
|
|
141
|
+
for group in comparison_groups(results):
|
|
142
|
+
python = group["python"]
|
|
143
|
+
backends = group["backends"]
|
|
144
|
+
cases = group["cases"]
|
|
145
|
+
sections.extend([f"### Python {python}", ""])
|
|
146
|
+
for result in group["sources"]:
|
|
147
|
+
environment = result["environment"]
|
|
148
|
+
implementation = environment.get("implementation", "CPython")
|
|
149
|
+
machine = result["machine_info"]
|
|
150
|
+
cpu = machine.get("cpu", {}).get("brand_raw", machine.get("processor", "unknown"))
|
|
151
|
+
sections.extend(
|
|
152
|
+
[
|
|
153
|
+
f"**{implementation} {environment['python']}**. Measured: {result['datetime']}. "
|
|
154
|
+
f"Revision: `{environment['revision']}`"
|
|
155
|
+
+ (" (working tree modified)." if environment["dirty"] else "."),
|
|
156
|
+
"",
|
|
157
|
+
f"Environment: {machine['system']} {machine['release']}, {machine['machine']}; {cpu}."
|
|
158
|
+
+ (
|
|
159
|
+
f" PyPy release: {machine['python_implementation_version']}."
|
|
160
|
+
if implementation == "PyPy" and "python_implementation_version" in machine
|
|
161
|
+
else ""
|
|
162
|
+
),
|
|
163
|
+
"",
|
|
164
|
+
"Dependencies: "
|
|
165
|
+
+ ", ".join(
|
|
166
|
+
f"`{key}=={value}`"
|
|
167
|
+
for key, value in sorted(environment["dependencies"].items())
|
|
168
|
+
)
|
|
169
|
+
+ ".",
|
|
170
|
+
"",
|
|
171
|
+
]
|
|
172
|
+
)
|
|
173
|
+
sections.extend(["#### Times and variation", ""])
|
|
174
|
+
slug = f"python-{python}"
|
|
175
|
+
if plots:
|
|
176
|
+
sections.extend(
|
|
177
|
+
[
|
|
178
|
+
f"",
|
|
179
|
+
"",
|
|
180
|
+
]
|
|
181
|
+
)
|
|
182
|
+
sections.extend(
|
|
183
|
+
[
|
|
184
|
+
"| Layer | Workload | Encoder | Median (µs) | IQR (µs) | Mean ± SD (µs) | Rounds × iterations |",
|
|
185
|
+
"| --- | --- | --- | ---: | ---: | ---: | ---: |",
|
|
186
|
+
]
|
|
187
|
+
)
|
|
188
|
+
for layer in LAYERS:
|
|
189
|
+
for workload in WORKLOADS:
|
|
190
|
+
for backend in backends:
|
|
191
|
+
stats = cases[layer, workload, backend]
|
|
192
|
+
sections.append(
|
|
193
|
+
f"| {layer} | {workload} | {backend} | "
|
|
194
|
+
f"{stats['median'] * 1e6:.3g} | {stats['iqr'] * 1e6:.2g} | "
|
|
195
|
+
f"{stats['mean'] * 1e6:.3g} ± {stats['stddev'] * 1e6:.2g} | "
|
|
196
|
+
f"{stats['rounds']:,} × {stats['iterations']:,} |"
|
|
197
|
+
)
|
|
198
|
+
sections.extend(["", "#### Speedup", ""])
|
|
199
|
+
if plots:
|
|
200
|
+
sections.extend(
|
|
201
|
+
[
|
|
202
|
+
f"",
|
|
203
|
+
"",
|
|
204
|
+
]
|
|
205
|
+
)
|
|
206
|
+
reference = "json (CPython)" if "json (CPython)" in backends else "json (PyPy)"
|
|
207
|
+
sections.extend([f"Reference: {reference} (1×).", ""])
|
|
208
|
+
sections.extend(
|
|
209
|
+
[
|
|
210
|
+
"| Layer | Workload | " + " | ".join(backends) + " |",
|
|
211
|
+
"| --- | --- | " + " | ".join("---:" for _ in backends) + " |",
|
|
212
|
+
]
|
|
213
|
+
)
|
|
214
|
+
for layer in LAYERS:
|
|
215
|
+
for workload in WORKLOADS:
|
|
216
|
+
baseline = cases[layer, workload, reference]["median"]
|
|
217
|
+
cells = []
|
|
218
|
+
for backend in backends:
|
|
219
|
+
timing = cases[layer, workload, backend]["median"]
|
|
220
|
+
cells.append(f"{baseline / timing:.2g}×")
|
|
221
|
+
sections.append(f"| {layer} | {workload} | " + " | ".join(cells) + " |")
|
|
222
|
+
sections.append("")
|
|
223
|
+
statistics = [benchmark["stats"] for result in results for benchmark in result["benchmarks"]]
|
|
224
|
+
rounds = [stats["rounds"] for stats in statistics]
|
|
225
|
+
iterations = [stats["iterations"] for stats in statistics]
|
|
226
|
+
variation = [stats["stddev"] / stats["mean"] * 100 for stats in statistics]
|
|
227
|
+
summary = [
|
|
228
|
+
f"Observed repetitions: {min(rounds):,}–{max(rounds):,} rounds per case, "
|
|
229
|
+
f"with {min(iterations):,}–{max(iterations):,} iterations per round. "
|
|
230
|
+
"Warmup and calibration calls are additional.",
|
|
231
|
+
"",
|
|
232
|
+
f"Within-run SD/mean: {min(variation):.2g}%–{max(variation):.2g}% across cases, "
|
|
233
|
+
"including outliers. This does not measure between-run variation.",
|
|
234
|
+
"",
|
|
235
|
+
]
|
|
236
|
+
lines = summary + sections
|
|
237
|
+
collapsed = []
|
|
238
|
+
in_table = False
|
|
239
|
+
for line in lines:
|
|
240
|
+
if line.startswith("|"):
|
|
241
|
+
if not in_table:
|
|
242
|
+
title = "Timing statistics" if "Median (µs)" in line else "Speedup values"
|
|
243
|
+
collapsed.extend([f'??? note "{title}"', ""])
|
|
244
|
+
collapsed.append(indent(line, " "))
|
|
245
|
+
in_table = True
|
|
246
|
+
else:
|
|
247
|
+
collapsed.append(line)
|
|
248
|
+
in_table = False
|
|
249
|
+
return "\n".join(collapsed)
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def main() -> None:
|
|
253
|
+
"""Print a summary or replace the generated section of a documentation page."""
|
|
254
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
255
|
+
parser.add_argument("results", nargs="+", type=Path)
|
|
256
|
+
output = parser.add_mutually_exclusive_group()
|
|
257
|
+
output.add_argument("--update", type=Path, help="Replace the generated section in this page")
|
|
258
|
+
output.add_argument("--curate", type=Path, help="Save JSON without individual timing samples")
|
|
259
|
+
parser.add_argument("--plots", help="Relative URL prefix for generated SVG plots")
|
|
260
|
+
args = parser.parse_args()
|
|
261
|
+
report = render(args.results, plots=args.plots)
|
|
262
|
+
if args.curate:
|
|
263
|
+
names = [path.name for path in args.results]
|
|
264
|
+
if len(set(names)) != len(names):
|
|
265
|
+
raise ValueError("Curated result filenames must be unique")
|
|
266
|
+
args.curate.mkdir(parents=True, exist_ok=True)
|
|
267
|
+
for path in args.results:
|
|
268
|
+
raw = path.read_bytes()
|
|
269
|
+
result = json.loads(raw)
|
|
270
|
+
for benchmark in result["benchmarks"]:
|
|
271
|
+
benchmark["stats"].pop("data", None)
|
|
272
|
+
result["curation"] = {
|
|
273
|
+
"source_sha256": hashlib.sha256(raw).hexdigest(),
|
|
274
|
+
"omitted": ["benchmarks[].stats.data"],
|
|
275
|
+
}
|
|
276
|
+
(args.curate / path.name).write_text(
|
|
277
|
+
json.dumps(result, indent=2) + "\n", encoding="utf-8"
|
|
278
|
+
)
|
|
279
|
+
elif args.update:
|
|
280
|
+
text = args.update.read_text(encoding="utf-8")
|
|
281
|
+
before, rest = text.split(START, 1)
|
|
282
|
+
_, after = rest.split(END, 1)
|
|
283
|
+
text = before + START + "\n\n" + report + "\n" + END + after
|
|
284
|
+
if SUMMARY_START in text:
|
|
285
|
+
before, rest = text.split(SUMMARY_START, 1)
|
|
286
|
+
_, after = rest.split(SUMMARY_END, 1)
|
|
287
|
+
summary = (
|
|
288
|
+
f"![Complete-formatting times and speedups by workload, with mean and version ranges]"
|
|
289
|
+
f"({args.plots}/summary.svg)\n"
|
|
290
|
+
if args.plots
|
|
291
|
+
else ""
|
|
292
|
+
)
|
|
293
|
+
text = before + SUMMARY_START + "\n\n" + summary + "\n" + SUMMARY_END + after
|
|
294
|
+
args.update.write_text(text, encoding="utf-8")
|
|
295
|
+
else:
|
|
296
|
+
print(report)
|
|
297
|
+
return
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
if __name__ == "__main__":
|
|
301
|
+
main()
|