pytest-better-report 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pytest_better_report-1.0.0/LICENSE +21 -0
- pytest_better_report-1.0.0/MANIFEST.in +3 -0
- pytest_better_report-1.0.0/PKG-INFO +163 -0
- pytest_better_report-1.0.0/README.md +138 -0
- pytest_better_report-1.0.0/pyproject.toml +62 -0
- pytest_better_report-1.0.0/pytest_better_report/__init__.py +3 -0
- pytest_better_report-1.0.0/pytest_better_report/compare.py +191 -0
- pytest_better_report-1.0.0/pytest_better_report/const.py +8 -0
- pytest_better_report-1.0.0/pytest_better_report/models/__init__.py +5 -0
- pytest_better_report-1.0.0/pytest_better_report/models/environment_data.py +7 -0
- pytest_better_report-1.0.0/pytest_better_report/models/execution_data.py +20 -0
- pytest_better_report-1.0.0/pytest_better_report/models/status.py +13 -0
- pytest_better_report-1.0.0/pytest_better_report/models/test_data.py +24 -0
- pytest_better_report-1.0.0/pytest_better_report/plugin.py +324 -0
- pytest_better_report-1.0.0/pytest_better_report/utils/__init__.py +0 -0
- pytest_better_report-1.0.0/pytest_better_report/utils/create_report.py +38 -0
- pytest_better_report-1.0.0/pytest_better_report/utils/helper.py +44 -0
- pytest_better_report-1.0.0/pytest_better_report/utils/pytest_helper.py +28 -0
- pytest_better_report-1.0.0/pytest_better_report.egg-info/PKG-INFO +163 -0
- pytest_better_report-1.0.0/pytest_better_report.egg-info/SOURCES.txt +23 -0
- pytest_better_report-1.0.0/pytest_better_report.egg-info/dependency_links.txt +1 -0
- pytest_better_report-1.0.0/pytest_better_report.egg-info/entry_points.txt +5 -0
- pytest_better_report-1.0.0/pytest_better_report.egg-info/requires.txt +4 -0
- pytest_better_report-1.0.0/pytest_better_report.egg-info/top_level.txt +1 -0
- pytest_better_report-1.0.0/setup.cfg +4 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Avi Zaguri
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: pytest-better-report
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: A pytest plugin that writes structured JSON (and optional Markdown) reports of every run, with CI metadata and a CLI to compare two runs.
|
|
5
|
+
Author: Avi Zaguri
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/aviz92/pytest-better-report
|
|
8
|
+
Project-URL: Repository, https://github.com/aviz92/pytest-better-report
|
|
9
|
+
Project-URL: Issues, https://github.com/aviz92/pytest-better-report/issues
|
|
10
|
+
Keywords: pytest,plugin,testing,report,json,markdown,ci
|
|
11
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
12
|
+
Classifier: Framework :: Pytest
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Topic :: Software Development :: Testing
|
|
15
|
+
Classifier: Operating System :: OS Independent
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Requires-Python: >=3.12
|
|
18
|
+
Description-Content-Type: text/markdown
|
|
19
|
+
License-File: LICENSE
|
|
20
|
+
Requires-Dist: custom-python-logger>=4.1.0
|
|
21
|
+
Requires-Dist: pytest>=9.2.0
|
|
22
|
+
Requires-Dist: python-base-command>=0.1.8
|
|
23
|
+
Requires-Dist: python-base-toolkit>=1.1.2
|
|
24
|
+
Dynamic: license-file
|
|
25
|
+
|
|
26
|
+

|
|
27
|
+

|
|
28
|
+

|
|
29
|
+

|
|
30
|
+

|
|
31
|
+

|
|
32
|
+
|
|
33
|
+
---
|
|
34
|
+
|
|
35
|
+
# 💡 pytest-better-report
|
|
36
|
+
A pytest plugin that writes a structured report of every run: a JSON file per test, a JSON file for the run with CI
|
|
37
|
+
metadata (repo, PR/MR, pipeline, commit), and an optional Markdown summary. Includes a CLI to compare two runs.
|
|
38
|
+
|
|
39
|
+
---
|
|
40
|
+
|
|
41
|
+
## 📦 Installation
|
|
42
|
+
```bash
|
|
43
|
+
uv add pytest-better-report
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
---
|
|
47
|
+
|
|
48
|
+
## 🚀 Features
|
|
49
|
+
- ✅ **Structured JSON** — `test_results.json` (one entry per test) and `execution_results.json` (the run)
|
|
50
|
+
- ✅ **CI traceability** — repo name, PR / MR number, pipeline number and commit in the run report
|
|
51
|
+
- ✅ **Markdown summary** — a table of every test with status, duration and message
|
|
52
|
+
- ✅ **Structured failure data** — raise with a JSON object as the message to record it as data
|
|
53
|
+
- ✅ **Compare two runs** — `better_report_compare` lists missing tests and status differences
|
|
54
|
+
- ✅ **Plays well with other plugins** — statuses come from pytest's final reports, whatever the plugin order
|
|
55
|
+
- ✅ **Zero footprint without the flag** — nothing is registered, no hook runs
|
|
56
|
+
|
|
57
|
+
---
|
|
58
|
+
|
|
59
|
+
## 🛠️ How to Use
|
|
60
|
+
```bash
|
|
61
|
+
pytest --better-report --md-report --pr-number=123 --commit="$GIT_SHA"
|
|
62
|
+
```
|
|
63
|
+
Files are written to `results_output/` (or `<output-dir>/results_output/`):
|
|
64
|
+
|
|
65
|
+
| File | Content |
|
|
66
|
+
|---|---|
|
|
67
|
+
| `test_results.json` | One entry per test: name, file, class, parameters, markers, status, times, exception, user properties |
|
|
68
|
+
| `execution_results.json` | The run: status, start / end / duration, CI metadata, environment, optionally the pytest command |
|
|
69
|
+
| `test_report.md` | With `--md-report`: a table of all tests and a summary by status |
|
|
70
|
+
|
|
71
|
+
---
|
|
72
|
+
|
|
73
|
+
## ⚙️ Flags
|
|
74
|
+
| Flag | Description |
|
|
75
|
+
|---|---|
|
|
76
|
+
| `--better-report` | Enable the report |
|
|
77
|
+
| `--output-dir=DIR` | Write the files to `DIR/results_output/` |
|
|
78
|
+
| `--md-report` | Also write `test_report.md` |
|
|
79
|
+
| `--traceback` | Add the crash location and traceback frames to each exception |
|
|
80
|
+
| `--repo-name`, `--pr-number`, `--mr-number`, `--pipeline-number`, `--commit` | CI metadata for `execution_results.json` |
|
|
81
|
+
| `--add-parameters` | Add each test parameter as its own field in `test_results.json` |
|
|
82
|
+
| `--pytest-command` | Record the pytest command line in `execution_results.json` |
|
|
83
|
+
| `--pytest-xfail-strict` | Treat an unexpected pass (XPASS) as a failure of the run |
|
|
84
|
+
| `--result-each-test` | Log each test's result after it runs |
|
|
85
|
+
| `--log-collected-tests` | Log every collected test at the start (works without `--better-report` too) |
|
|
86
|
+
|
|
87
|
+
---
|
|
88
|
+
|
|
89
|
+
## 📊 Statuses
|
|
90
|
+
Each test gets one status in `test_results.json`.
|
|
91
|
+
The run as a whole gets `execution_info.execution_status` in `execution_results.json`: `failed` if **at least one**
|
|
92
|
+
test has a status marked "Yes" in the last column, otherwise `passed`.
|
|
93
|
+
|
|
94
|
+
This is the report's own verdict on the run, written into the file. It is separate from pytest's exit code, which this
|
|
95
|
+
plugin doesn't change.
|
|
96
|
+
|
|
97
|
+
| Test status | Meaning | Makes `execution_status` = `failed` |
|
|
98
|
+
|---|---|---|
|
|
99
|
+
| `passed` | Passed | No |
|
|
100
|
+
| `failed` | Failed or errored in setup, call or teardown | Yes |
|
|
101
|
+
| `skipped` | Skipped (marker, fixture, or a plugin that converted the outcome) | No |
|
|
102
|
+
| `xfailed` | Expected failure | No |
|
|
103
|
+
| `xpassed` | Unexpected pass | Only with `--pytest-xfail-strict` |
|
|
104
|
+
| `collected` | Collected but not run (e.g. the run stopped early) | No |
|
|
105
|
+
|
|
106
|
+
---
|
|
107
|
+
|
|
108
|
+
## ▶️ Usage Examples
|
|
109
|
+
### Record structured failure data
|
|
110
|
+
```python
|
|
111
|
+
import json
|
|
112
|
+
|
|
113
|
+
def test_api():
|
|
114
|
+
response = call_api()
|
|
115
|
+
if response.status != 200:
|
|
116
|
+
raise AssertionError(json.dumps({"status": response.status, "body": response.text}))
|
|
117
|
+
```
|
|
118
|
+
The exception is recorded as `{"status": ..., "body": ...}` instead of plain text.
|
|
119
|
+
|
|
120
|
+
### Compare two runs
|
|
121
|
+
```bash
|
|
122
|
+
better_report_compare -a main/test_results.json -b branch/test_results.json --fail-on-diff
|
|
123
|
+
```
|
|
124
|
+
Lists tests missing from either side and tests whose status changed; exits non-zero with `--fail-on-diff` when
|
|
125
|
+
there are differences. `--test-file-name` limits the comparison to one test file.
|
|
126
|
+
|
|
127
|
+
---
|
|
128
|
+
|
|
129
|
+
## 🔁 Migrating from pytest-plugins
|
|
130
|
+
This plugin was previously `better_report` in `pytest-plugins`. Flags, file names and the JSON schema are unchanged.
|
|
131
|
+
Both register the same options, so don't install them together until you're on a `pytest-plugins` release without it.
|
|
132
|
+
Differences you may notice:
|
|
133
|
+
- A test that errors in teardown is now `failed` (it was reported as passed).
|
|
134
|
+
- The `failed-skipped` status no longer exists; converted failures are `skipped`, with the reason in `test_user_properties`.
|
|
135
|
+
- The logger is named `pytest_better_report`.
|
|
136
|
+
|
|
137
|
+
---
|
|
138
|
+
|
|
139
|
+
## 🤝 Contributing
|
|
140
|
+
1. Fork the repository
|
|
141
|
+
2. Create a feature branch: `git checkout -b feature/my-addition`
|
|
142
|
+
3. Make your changes
|
|
143
|
+
4. Commit your changes: `git commit -m 'Add my addition'`
|
|
144
|
+
5. Push to the branch: `git push origin feature/my-addition`
|
|
145
|
+
6. Open a Pull Request
|
|
146
|
+
|
|
147
|
+
Additions that promote clean, productive, and maintainable development are always welcome.
|
|
148
|
+
|
|
149
|
+
---
|
|
150
|
+
|
|
151
|
+
## 📄 License
|
|
152
|
+
MIT License — see [LICENSE](LICENSE) for details.
|
|
153
|
+
|
|
154
|
+
---
|
|
155
|
+
|
|
156
|
+
## 🙏 Thanks
|
|
157
|
+
Thanks for exploring this repository! <br>
|
|
158
|
+
Happy coding!
|
|
159
|
+
|
|
160
|
+
[](https://github.com/aviz92)
|
|
161
|
+
[](https://pypi.org/user/aviz/)
|
|
162
|
+
[](https://aviz92.github.io/)
|
|
163
|
+
[](https://www.linkedin.com/in/avi-zaguri-41869b11b)
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+

|
|
2
|
+

|
|
3
|
+

|
|
4
|
+

|
|
5
|
+

|
|
6
|
+

|
|
7
|
+
|
|
8
|
+
---
|
|
9
|
+
|
|
10
|
+
# 💡 pytest-better-report
|
|
11
|
+
A pytest plugin that writes a structured report of every run: a JSON file per test, a JSON file for the run with CI
|
|
12
|
+
metadata (repo, PR/MR, pipeline, commit), and an optional Markdown summary. Includes a CLI to compare two runs.
|
|
13
|
+
|
|
14
|
+
---
|
|
15
|
+
|
|
16
|
+
## 📦 Installation
|
|
17
|
+
```bash
|
|
18
|
+
uv add pytest-better-report
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
---
|
|
22
|
+
|
|
23
|
+
## 🚀 Features
|
|
24
|
+
- ✅ **Structured JSON** — `test_results.json` (one entry per test) and `execution_results.json` (the run)
|
|
25
|
+
- ✅ **CI traceability** — repo name, PR / MR number, pipeline number and commit in the run report
|
|
26
|
+
- ✅ **Markdown summary** — a table of every test with status, duration and message
|
|
27
|
+
- ✅ **Structured failure data** — raise with a JSON object as the message to record it as data
|
|
28
|
+
- ✅ **Compare two runs** — `better_report_compare` lists missing tests and status differences
|
|
29
|
+
- ✅ **Plays well with other plugins** — statuses come from pytest's final reports, whatever the plugin order
|
|
30
|
+
- ✅ **Zero footprint without the flag** — nothing is registered, no hook runs
|
|
31
|
+
|
|
32
|
+
---
|
|
33
|
+
|
|
34
|
+
## 🛠️ How to Use
|
|
35
|
+
```bash
|
|
36
|
+
pytest --better-report --md-report --pr-number=123 --commit="$GIT_SHA"
|
|
37
|
+
```
|
|
38
|
+
Files are written to `results_output/` (or `<output-dir>/results_output/`):
|
|
39
|
+
|
|
40
|
+
| File | Content |
|
|
41
|
+
|---|---|
|
|
42
|
+
| `test_results.json` | One entry per test: name, file, class, parameters, markers, status, times, exception, user properties |
|
|
43
|
+
| `execution_results.json` | The run: status, start / end / duration, CI metadata, environment, optionally the pytest command |
|
|
44
|
+
| `test_report.md` | With `--md-report`: a table of all tests and a summary by status |
|
|
45
|
+
|
|
46
|
+
---
|
|
47
|
+
|
|
48
|
+
## ⚙️ Flags
|
|
49
|
+
| Flag | Description |
|
|
50
|
+
|---|---|
|
|
51
|
+
| `--better-report` | Enable the report |
|
|
52
|
+
| `--output-dir=DIR` | Write the files to `DIR/results_output/` |
|
|
53
|
+
| `--md-report` | Also write `test_report.md` |
|
|
54
|
+
| `--traceback` | Add the crash location and traceback frames to each exception |
|
|
55
|
+
| `--repo-name`, `--pr-number`, `--mr-number`, `--pipeline-number`, `--commit` | CI metadata for `execution_results.json` |
|
|
56
|
+
| `--add-parameters` | Add each test parameter as its own field in `test_results.json` |
|
|
57
|
+
| `--pytest-command` | Record the pytest command line in `execution_results.json` |
|
|
58
|
+
| `--pytest-xfail-strict` | Treat an unexpected pass (XPASS) as a failure of the run |
|
|
59
|
+
| `--result-each-test` | Log each test's result after it runs |
|
|
60
|
+
| `--log-collected-tests` | Log every collected test at the start (works without `--better-report` too) |
|
|
61
|
+
|
|
62
|
+
---
|
|
63
|
+
|
|
64
|
+
## 📊 Statuses
|
|
65
|
+
Each test gets one status in `test_results.json`.
|
|
66
|
+
The run as a whole gets `execution_info.execution_status` in `execution_results.json`: `failed` if **at least one**
|
|
67
|
+
test has a status marked "Yes" in the last column, otherwise `passed`.
|
|
68
|
+
|
|
69
|
+
This is the report's own verdict on the run, written into the file. It is separate from pytest's exit code, which this
|
|
70
|
+
plugin doesn't change.
|
|
71
|
+
|
|
72
|
+
| Test status | Meaning | Makes `execution_status` = `failed` |
|
|
73
|
+
|---|---|---|
|
|
74
|
+
| `passed` | Passed | No |
|
|
75
|
+
| `failed` | Failed or errored in setup, call or teardown | Yes |
|
|
76
|
+
| `skipped` | Skipped (marker, fixture, or a plugin that converted the outcome) | No |
|
|
77
|
+
| `xfailed` | Expected failure | No |
|
|
78
|
+
| `xpassed` | Unexpected pass | Only with `--pytest-xfail-strict` |
|
|
79
|
+
| `collected` | Collected but not run (e.g. the run stopped early) | No |
|
|
80
|
+
|
|
81
|
+
---
|
|
82
|
+
|
|
83
|
+
## ▶️ Usage Examples
|
|
84
|
+
### Record structured failure data
|
|
85
|
+
```python
|
|
86
|
+
import json
|
|
87
|
+
|
|
88
|
+
def test_api():
|
|
89
|
+
response = call_api()
|
|
90
|
+
if response.status != 200:
|
|
91
|
+
raise AssertionError(json.dumps({"status": response.status, "body": response.text}))
|
|
92
|
+
```
|
|
93
|
+
The exception is recorded as `{"status": ..., "body": ...}` instead of plain text.
|
|
94
|
+
|
|
95
|
+
### Compare two runs
|
|
96
|
+
```bash
|
|
97
|
+
better_report_compare -a main/test_results.json -b branch/test_results.json --fail-on-diff
|
|
98
|
+
```
|
|
99
|
+
Lists tests missing from either side and tests whose status changed; exits non-zero with `--fail-on-diff` when
|
|
100
|
+
there are differences. `--test-file-name` limits the comparison to one test file.
|
|
101
|
+
|
|
102
|
+
---
|
|
103
|
+
|
|
104
|
+
## 🔁 Migrating from pytest-plugins
|
|
105
|
+
This plugin was previously `better_report` in `pytest-plugins`. Flags, file names and the JSON schema are unchanged.
|
|
106
|
+
Both register the same options, so don't install them together until you're on a `pytest-plugins` release without it.
|
|
107
|
+
Differences you may notice:
|
|
108
|
+
- A test that errors in teardown is now `failed` (it was reported as passed).
|
|
109
|
+
- The `failed-skipped` status no longer exists; converted failures are `skipped`, with the reason in `test_user_properties`.
|
|
110
|
+
- The logger is named `pytest_better_report`.
|
|
111
|
+
|
|
112
|
+
---
|
|
113
|
+
|
|
114
|
+
## 🤝 Contributing
|
|
115
|
+
1. Fork the repository
|
|
116
|
+
2. Create a feature branch: `git checkout -b feature/my-addition`
|
|
117
|
+
3. Make your changes
|
|
118
|
+
4. Commit your changes: `git commit -m 'Add my addition'`
|
|
119
|
+
5. Push to the branch: `git push origin feature/my-addition`
|
|
120
|
+
6. Open a Pull Request
|
|
121
|
+
|
|
122
|
+
Additions that promote clean, productive, and maintainable development are always welcome.
|
|
123
|
+
|
|
124
|
+
---
|
|
125
|
+
|
|
126
|
+
## 📄 License
|
|
127
|
+
MIT License — see [LICENSE](LICENSE) for details.
|
|
128
|
+
|
|
129
|
+
---
|
|
130
|
+
|
|
131
|
+
## 🙏 Thanks
|
|
132
|
+
Thanks for exploring this repository! <br>
|
|
133
|
+
Happy coding!
|
|
134
|
+
|
|
135
|
+
[](https://github.com/aviz92)
|
|
136
|
+
[](https://pypi.org/user/aviz/)
|
|
137
|
+
[](https://aviz92.github.io/)
|
|
138
|
+
[](https://www.linkedin.com/in/avi-zaguri-41869b11b)
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "pytest-better-report"
|
|
7
|
+
version = "1.0.0"
|
|
8
|
+
description = "A pytest plugin that writes structured JSON (and optional Markdown) reports of every run, with CI metadata and a CLI to compare two runs."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.12"
|
|
11
|
+
license = { text = "MIT" }
|
|
12
|
+
authors = [{ name = "Avi Zaguri" },]
|
|
13
|
+
|
|
14
|
+
keywords = [
|
|
15
|
+
"pytest",
|
|
16
|
+
"plugin",
|
|
17
|
+
"testing",
|
|
18
|
+
"report",
|
|
19
|
+
"json",
|
|
20
|
+
"markdown",
|
|
21
|
+
"ci",
|
|
22
|
+
]
|
|
23
|
+
|
|
24
|
+
classifiers = [
|
|
25
|
+
"Development Status :: 5 - Production/Stable",
|
|
26
|
+
"Framework :: Pytest",
|
|
27
|
+
"Intended Audience :: Developers",
|
|
28
|
+
"Topic :: Software Development :: Testing",
|
|
29
|
+
"Operating System :: OS Independent",
|
|
30
|
+
"Programming Language :: Python :: 3.12",
|
|
31
|
+
]
|
|
32
|
+
|
|
33
|
+
dependencies = [
|
|
34
|
+
"custom-python-logger>=4.1.0",
|
|
35
|
+
"pytest>=9.2.0",
|
|
36
|
+
"python-base-command>=0.1.8",
|
|
37
|
+
"python-base-toolkit>=1.1.2",
|
|
38
|
+
]
|
|
39
|
+
|
|
40
|
+
[dependency-groups]
|
|
41
|
+
dev = [
|
|
42
|
+
"pre-commit>=4.7.0",
|
|
43
|
+
"python-base-command>=0.1.8",
|
|
44
|
+
"setuptools>=84.0.0",
|
|
45
|
+
"wheel>=0.45.1",
|
|
46
|
+
]
|
|
47
|
+
|
|
48
|
+
[project.urls]
|
|
49
|
+
Homepage = "https://github.com/aviz92/pytest-better-report"
|
|
50
|
+
Repository = "https://github.com/aviz92/pytest-better-report"
|
|
51
|
+
Issues = "https://github.com/aviz92/pytest-better-report/issues"
|
|
52
|
+
|
|
53
|
+
[tool.setuptools.packages.find]
|
|
54
|
+
where = ["."]
|
|
55
|
+
include = ["pytest_better_report*"]
|
|
56
|
+
namespaces = false
|
|
57
|
+
|
|
58
|
+
[project.entry-points.pytest11]
|
|
59
|
+
better_report = "pytest_better_report.plugin"
|
|
60
|
+
|
|
61
|
+
[project.scripts]
|
|
62
|
+
better_report_compare = "pytest_better_report.compare:main"
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
"""
|
|
2
|
+
CLI tool to compare two better-report JSON log files.
|
|
3
|
+
|
|
4
|
+
Checks if all pytest_test_name entries are equal, lists missing tests from each file,
|
|
5
|
+
and for matching tests compares their test_status and lists all diffs.
|
|
6
|
+
|
|
7
|
+
Usage:
|
|
8
|
+
python better_report_compare.py -a <file_a.json> -b <file_b.json> [--fail-on-diff]
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
import os
|
|
13
|
+
import sys
|
|
14
|
+
from dataclasses import dataclass
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from typing import Any
|
|
17
|
+
|
|
18
|
+
from python_base_command import BaseCommand, CommandError, CommandParser
|
|
19
|
+
|
|
20
|
+
PASS_SYMBOL = "✓"
|
|
21
|
+
FAIL_SYMBOL = "✗"
|
|
22
|
+
DIFF_SYMBOL = "≠"
|
|
23
|
+
|
|
24
|
+
SEP = "─" * 100
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _log_missing(missing: list[str], source: str, target: str) -> None:
|
|
28
|
+
if missing:
|
|
29
|
+
print(f"\n{FAIL_SYMBOL} Tests in {source} but MISSING from {target} ({len(missing)})")
|
|
30
|
+
for name in missing:
|
|
31
|
+
print(f" - {name}")
|
|
32
|
+
else:
|
|
33
|
+
print(f"\n{PASS_SYMBOL} No tests missing from {target}")
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _log_status_diffs(status_diffs: list[tuple[str, str, str]]) -> None:
|
|
37
|
+
if status_diffs:
|
|
38
|
+
col_w = max(len(name) for name, _, _ in status_diffs)
|
|
39
|
+
print(f"\n{DIFF_SYMBOL} Status differences in common tests ({len(status_diffs)})")
|
|
40
|
+
print(f" {'TEST NAME':<{col_w}} {'STATUS IN A':<12} STATUS IN B")
|
|
41
|
+
print(" " + "-" * (col_w + 28))
|
|
42
|
+
for name, s_a, s_b in status_diffs:
|
|
43
|
+
print(f" {name:<{col_w}} {s_a:<12} {s_b}")
|
|
44
|
+
else:
|
|
45
|
+
print(f"\n{PASS_SYMBOL} All common tests have identical statuses")
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@dataclass(frozen=True)
|
|
49
|
+
class ComparisonSummary:
|
|
50
|
+
file_a: Path
|
|
51
|
+
file_b: Path
|
|
52
|
+
total_a: int
|
|
53
|
+
total_b: int
|
|
54
|
+
common_count: int
|
|
55
|
+
only_in_a: list[str]
|
|
56
|
+
only_in_b: list[str]
|
|
57
|
+
status_diffs: list[tuple[str, str, str]]
|
|
58
|
+
test_file_name: str | None = None
|
|
59
|
+
|
|
60
|
+
@property
|
|
61
|
+
def identical(self) -> bool:
|
|
62
|
+
return not (self.only_in_a or self.only_in_b or self.status_diffs)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _log_summary(summary: ComparisonSummary) -> None:
|
|
66
|
+
print(f"\n{SEP}")
|
|
67
|
+
print(" BETTER-REPORT COMPARISON")
|
|
68
|
+
print(SEP)
|
|
69
|
+
print(f" File A : {summary.file_a}")
|
|
70
|
+
print(f" File B : {summary.file_b}")
|
|
71
|
+
if summary.test_file_name:
|
|
72
|
+
print(f" Filter : test_file_name = {summary.test_file_name}")
|
|
73
|
+
print(f" Tests : {summary.total_a} in A | {summary.total_b} in B | {summary.common_count} in common")
|
|
74
|
+
print(SEP)
|
|
75
|
+
|
|
76
|
+
_log_missing(summary.only_in_a, source="A", target="B")
|
|
77
|
+
_log_missing(summary.only_in_b, source="B", target="A")
|
|
78
|
+
_log_status_diffs(summary.status_diffs)
|
|
79
|
+
|
|
80
|
+
print(f"\n{SEP}")
|
|
81
|
+
if not summary.identical:
|
|
82
|
+
print(" RESULT: DIFFERENCES FOUND")
|
|
83
|
+
else:
|
|
84
|
+
print(" RESULT: REPORTS ARE IDENTICAL")
|
|
85
|
+
print(f"{SEP}\n")
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def load_report(path: Path, test_file_name: str | None = None) -> dict[str, dict]:
|
|
89
|
+
with open(path) as f:
|
|
90
|
+
raw = json.load(f)
|
|
91
|
+
indexed: dict[str, dict] = {}
|
|
92
|
+
for entry in raw.values():
|
|
93
|
+
if test_file_name and entry.get("test_file_name") != test_file_name:
|
|
94
|
+
continue
|
|
95
|
+
if pytest_name := entry.get("pytest_test_name"):
|
|
96
|
+
indexed[pytest_name] = entry
|
|
97
|
+
return indexed
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def compare_reports(file_a: Path, file_b: Path, test_file_name: str | None = None) -> bool:
|
|
101
|
+
report_a = load_report(file_a, test_file_name)
|
|
102
|
+
report_b = load_report(file_b, test_file_name)
|
|
103
|
+
|
|
104
|
+
names_a = set(report_a.keys())
|
|
105
|
+
names_b = set(report_b.keys())
|
|
106
|
+
|
|
107
|
+
only_in_a = sorted(names_a - names_b)
|
|
108
|
+
only_in_b = sorted(names_b - names_a)
|
|
109
|
+
common = sorted(names_a & names_b)
|
|
110
|
+
|
|
111
|
+
status_diffs = [
|
|
112
|
+
(name, report_a[name]["test_status"], report_b[name]["test_status"])
|
|
113
|
+
for name in common
|
|
114
|
+
if report_a[name].get("test_status") != report_b[name].get("test_status")
|
|
115
|
+
]
|
|
116
|
+
|
|
117
|
+
summary = ComparisonSummary(
|
|
118
|
+
file_a=file_a,
|
|
119
|
+
file_b=file_b,
|
|
120
|
+
total_a=len(names_a),
|
|
121
|
+
total_b=len(names_b),
|
|
122
|
+
common_count=len(common),
|
|
123
|
+
only_in_a=only_in_a,
|
|
124
|
+
only_in_b=only_in_b,
|
|
125
|
+
status_diffs=status_diffs,
|
|
126
|
+
test_file_name=test_file_name,
|
|
127
|
+
)
|
|
128
|
+
_log_summary(summary=summary)
|
|
129
|
+
return summary.identical
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
class Command(BaseCommand):
|
|
133
|
+
help = "Compare two better-report JSON log files for test coverage and status differences."
|
|
134
|
+
version = "1.0.0"
|
|
135
|
+
|
|
136
|
+
def add_arguments(self, parser: CommandParser) -> None:
|
|
137
|
+
parser.add_argument(
|
|
138
|
+
"-a",
|
|
139
|
+
"--file-a",
|
|
140
|
+
type=Path,
|
|
141
|
+
required=True,
|
|
142
|
+
help="Path to the first better-report JSON file.",
|
|
143
|
+
)
|
|
144
|
+
parser.add_argument(
|
|
145
|
+
"-b",
|
|
146
|
+
"--file-b",
|
|
147
|
+
type=Path,
|
|
148
|
+
required=True,
|
|
149
|
+
help="Path to the second better-report JSON file.",
|
|
150
|
+
)
|
|
151
|
+
parser.add_argument(
|
|
152
|
+
"--fail-on-diff",
|
|
153
|
+
action="store_true",
|
|
154
|
+
default=False,
|
|
155
|
+
help="Exit with non-zero code if any differences are found.",
|
|
156
|
+
)
|
|
157
|
+
parser.add_argument(
|
|
158
|
+
"--test-file-name",
|
|
159
|
+
type=str,
|
|
160
|
+
required=False,
|
|
161
|
+
default=None,
|
|
162
|
+
help="Filter comparison to tests belonging to this test_file_name "
|
|
163
|
+
"(e.g. test_advertiser_services_missing.py).",
|
|
164
|
+
)
|
|
165
|
+
|
|
166
|
+
def handle(self, **kwargs: Any) -> None:
|
|
167
|
+
file_a: Path = kwargs["file_a"]
|
|
168
|
+
file_b: Path = kwargs["file_b"]
|
|
169
|
+
fail_on_diff: bool = kwargs["fail_on_diff"]
|
|
170
|
+
test_file_name: str | None = kwargs.get("test_file_name")
|
|
171
|
+
|
|
172
|
+
for label, path in (("A", file_a), ("B", file_b)):
|
|
173
|
+
if not path.exists():
|
|
174
|
+
raise FileNotFoundError(f"File {label} not found: {path}")
|
|
175
|
+
|
|
176
|
+
identical = compare_reports(file_a, file_b, test_file_name)
|
|
177
|
+
|
|
178
|
+
if fail_on_diff and not identical:
|
|
179
|
+
raise CommandError("Reports have differences.")
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def main() -> None:
|
|
183
|
+
"""Entry point: run the compare command from CLI."""
|
|
184
|
+
print('Starting "Better Report Compare"')
|
|
185
|
+
|
|
186
|
+
os.environ["PYTHON_BASE_COMMAND_LOG_FILE"] = "false"
|
|
187
|
+
Command().run_from_argv(argv=sys.argv)
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
if __name__ == "__main__":
|
|
191
|
+
main()
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
LOGGER_NAME = "pytest_better_report"
|
|
2
|
+
|
|
3
|
+
EXECUTION_RESULTS_FILENAME = "execution_results.json"
|
|
4
|
+
TEST_RESULTS_FILENAME = "test_results.json"
|
|
5
|
+
MD_REPORT_FILENAME = "test_report.md"
|
|
6
|
+
OUTPUT_DIR_NAME = "results_output"
|
|
7
|
+
|
|
8
|
+
REPORTER_PLUGIN_NAME = "better-report-reporter"
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
|
|
3
|
+
from pytest_better_report.models.status import ExecutionStatus
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
@dataclass
|
|
7
|
+
class ExecutionData:
|
|
8
|
+
execution_status: ExecutionStatus
|
|
9
|
+
revision: str | None
|
|
10
|
+
execution_start_time: str | None = None
|
|
11
|
+
execution_end_time: str | None = None
|
|
12
|
+
execution_duration_sec: str | None = None
|
|
13
|
+
|
|
14
|
+
repo_name: str | None = None
|
|
15
|
+
pull_request_number: str | None = None
|
|
16
|
+
merge_request_number: str | None = None
|
|
17
|
+
pipeline_number: str | None = None
|
|
18
|
+
commit: str | None = None
|
|
19
|
+
|
|
20
|
+
test_list: list | None = None
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
from enum import StrEnum
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class ExecutionStatus(StrEnum):
|
|
5
|
+
COLLECTED = "collected"
|
|
6
|
+
STARTED = "started"
|
|
7
|
+
PENDING = "pending"
|
|
8
|
+
PASSED = "passed"
|
|
9
|
+
FAILED = "failed"
|
|
10
|
+
XFAIL = "xfailed"
|
|
11
|
+
XPASS = "xpassed"
|
|
12
|
+
CANCELLED = "cancelled"
|
|
13
|
+
SKIPPED = "skipped"
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
|
|
3
|
+
from pytest_better_report.models.status import ExecutionStatus
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
@dataclass
|
|
7
|
+
class TestData:
|
|
8
|
+
__test__ = False # a data model, not a test class (stops pytest from trying to collect it)
|
|
9
|
+
|
|
10
|
+
test_file_name: str
|
|
11
|
+
class_test_name: str | None
|
|
12
|
+
test_name: str
|
|
13
|
+
pytest_test_name: str
|
|
14
|
+
test_full_name: str
|
|
15
|
+
test_full_path: str
|
|
16
|
+
test_status: ExecutionStatus = ExecutionStatus.COLLECTED
|
|
17
|
+
test_parameters: dict[str, object] | None = None
|
|
18
|
+
test_markers: list | None = None
|
|
19
|
+
test_start_time: str | None = None
|
|
20
|
+
test_end_time: str | None = None
|
|
21
|
+
test_duration_sec: float | None = None # only for the tst itself, not including fixtures
|
|
22
|
+
exception_message: dict | None = None
|
|
23
|
+
run_index: int | None = None
|
|
24
|
+
test_user_properties: dict[str, object] | None = None
|