mentistest-coverage 0.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mentistest_coverage-0.2.2/CHANGELOG.md +34 -0
- mentistest_coverage-0.2.2/LICENSE +21 -0
- mentistest_coverage-0.2.2/MANIFEST.in +2 -0
- mentistest_coverage-0.2.2/PKG-INFO +310 -0
- mentistest_coverage-0.2.2/README.md +216 -0
- mentistest_coverage-0.2.2/mentistest_coverage.egg-info/PKG-INFO +310 -0
- mentistest_coverage-0.2.2/mentistest_coverage.egg-info/SOURCES.txt +93 -0
- mentistest_coverage-0.2.2/mentistest_coverage.egg-info/dependency_links.txt +1 -0
- mentistest_coverage-0.2.2/mentistest_coverage.egg-info/entry_points.txt +2 -0
- mentistest_coverage-0.2.2/mentistest_coverage.egg-info/requires.txt +62 -0
- mentistest_coverage-0.2.2/mentistest_coverage.egg-info/top_level.txt +1 -0
- mentistest_coverage-0.2.2/pyproject.toml +374 -0
- mentistest_coverage-0.2.2/setup.cfg +4 -0
- mentistest_coverage-0.2.2/test_coverage_tool/__init__.py +17 -0
- mentistest_coverage-0.2.2/test_coverage_tool/_logging.py +34 -0
- mentistest_coverage-0.2.2/test_coverage_tool/_version.py +4 -0
- mentistest_coverage-0.2.2/test_coverage_tool/ai/__init__.py +8 -0
- mentistest_coverage-0.2.2/test_coverage_tool/ai/_scrub.py +83 -0
- mentistest_coverage-0.2.2/test_coverage_tool/ai/analyzer.py +455 -0
- mentistest_coverage-0.2.2/test_coverage_tool/ai/budget.py +177 -0
- mentistest_coverage-0.2.2/test_coverage_tool/ai/config.py +93 -0
- mentistest_coverage-0.2.2/test_coverage_tool/api/__init__.py +7 -0
- mentistest_coverage-0.2.2/test_coverage_tool/api/_logging_middleware.py +218 -0
- mentistest_coverage-0.2.2/test_coverage_tool/api/_metrics.py +131 -0
- mentistest_coverage-0.2.2/test_coverage_tool/api/_security.py +284 -0
- mentistest_coverage-0.2.2/test_coverage_tool/api/models.py +104 -0
- mentistest_coverage-0.2.2/test_coverage_tool/api/routes.py +661 -0
- mentistest_coverage-0.2.2/test_coverage_tool/benchmarks/__init__.py +7 -0
- mentistest_coverage-0.2.2/test_coverage_tool/benchmarks/_templates/compare.html.jinja +219 -0
- mentistest_coverage-0.2.2/test_coverage_tool/benchmarks/_templates/dashboard.html.jinja +977 -0
- mentistest_coverage-0.2.2/test_coverage_tool/benchmarks/dashboard.py +706 -0
- mentistest_coverage-0.2.2/test_coverage_tool/benchmarks/store.py +520 -0
- mentistest_coverage-0.2.2/test_coverage_tool/cli/__init__.py +7 -0
- mentistest_coverage-0.2.2/test_coverage_tool/cli/main.py +706 -0
- mentistest_coverage-0.2.2/test_coverage_tool/core/__init__.py +14 -0
- mentistest_coverage-0.2.2/test_coverage_tool/core/analyzer.py +1194 -0
- mentistest_coverage-0.2.2/test_coverage_tool/core/call_graph.py +240 -0
- mentistest_coverage-0.2.2/test_coverage_tool/core/classifier.py +982 -0
- mentistest_coverage-0.2.2/test_coverage_tool/core/classifier_facts.py +676 -0
- mentistest_coverage-0.2.2/test_coverage_tool/core/classifier_validation.py +400 -0
- mentistest_coverage-0.2.2/test_coverage_tool/core/contract_coverage.py +101 -0
- mentistest_coverage-0.2.2/test_coverage_tool/core/diff_coverage.py +226 -0
- mentistest_coverage-0.2.2/test_coverage_tool/core/endpoint_attribution.py +303 -0
- mentistest_coverage-0.2.2/test_coverage_tool/core/endpoint_correlation.py +364 -0
- mentistest_coverage-0.2.2/test_coverage_tool/core/endpoint_extractors/__init__.py +56 -0
- mentistest_coverage-0.2.2/test_coverage_tool/core/endpoint_extractors/class_views.py +469 -0
- mentistest_coverage-0.2.2/test_coverage_tool/core/endpoint_extractors/cross_check.py +205 -0
- mentistest_coverage-0.2.2/test_coverage_tool/core/endpoint_extractors/inbound.py +413 -0
- mentistest_coverage-0.2.2/test_coverage_tool/core/endpoint_extractors/javascript.py +335 -0
- mentistest_coverage-0.2.2/test_coverage_tool/core/endpoint_extractors/mock_libs.py +170 -0
- mentistest_coverage-0.2.2/test_coverage_tool/core/endpoint_extractors/outbound.py +440 -0
- mentistest_coverage-0.2.2/test_coverage_tool/core/endpoint_extractors/overrides.py +228 -0
- mentistest_coverage-0.2.2/test_coverage_tool/core/endpoint_extractors/wrappers.py +162 -0
- mentistest_coverage-0.2.2/test_coverage_tool/core/fixture_resolver.py +287 -0
- mentistest_coverage-0.2.2/test_coverage_tool/core/git.py +196 -0
- mentistest_coverage-0.2.2/test_coverage_tool/core/imports.py +101 -0
- mentistest_coverage-0.2.2/test_coverage_tool/core/javascript_classifier.py +273 -0
- mentistest_coverage-0.2.2/test_coverage_tool/core/marker_config.py +251 -0
- mentistest_coverage-0.2.2/test_coverage_tool/core/metrics.py +332 -0
- mentistest_coverage-0.2.2/test_coverage_tool/core/reclassify.py +503 -0
- mentistest_coverage-0.2.2/test_coverage_tool/core/regression.py +184 -0
- mentistest_coverage-0.2.2/test_coverage_tool/core/symbol_table.py +348 -0
- mentistest_coverage-0.2.2/test_coverage_tool/core/test_type.py +52 -0
- mentistest_coverage-0.2.2/test_coverage_tool/core/timing.py +223 -0
- mentistest_coverage-0.2.2/test_coverage_tool/core/types.py +1038 -0
- mentistest_coverage-0.2.2/test_coverage_tool/history/__init__.py +5 -0
- mentistest_coverage-0.2.2/test_coverage_tool/history/backfill.py +369 -0
- mentistest_coverage-0.2.2/test_coverage_tool/history/comparison.py +297 -0
- mentistest_coverage-0.2.2/test_coverage_tool/history/comparison_dashboard.py +353 -0
- mentistest_coverage-0.2.2/test_coverage_tool/history/dashboard.py +794 -0
- mentistest_coverage-0.2.2/test_coverage_tool/history/store.py +588 -0
- mentistest_coverage-0.2.2/test_coverage_tool/parsers/__init__.py +9 -0
- mentistest_coverage-0.2.2/test_coverage_tool/parsers/_constants.py +13 -0
- mentistest_coverage-0.2.2/test_coverage_tool/parsers/code_parser.py +201 -0
- mentistest_coverage-0.2.2/test_coverage_tool/parsers/coverage_contexts.py +172 -0
- mentistest_coverage-0.2.2/test_coverage_tool/parsers/coverage_parser.py +180 -0
- mentistest_coverage-0.2.2/test_coverage_tool/parsers/cypress_parser.py +95 -0
- mentistest_coverage-0.2.2/test_coverage_tool/parsers/http_traffic_parser.py +143 -0
- mentistest_coverage-0.2.2/test_coverage_tool/parsers/istanbul_parser.py +172 -0
- mentistest_coverage-0.2.2/test_coverage_tool/parsers/mutation_parser.py +209 -0
- mentistest_coverage-0.2.2/test_coverage_tool/parsers/pact_parser.py +227 -0
- mentistest_coverage-0.2.2/test_coverage_tool/parsers/spec_parser.py +118 -0
- mentistest_coverage-0.2.2/test_coverage_tool/parsers/test_parser.py +335 -0
- mentistest_coverage-0.2.2/test_coverage_tool/portfolio/__init__.py +18 -0
- mentistest_coverage-0.2.2/test_coverage_tool/portfolio/_templates/portfolio.html.jinja +130 -0
- mentistest_coverage-0.2.2/test_coverage_tool/portfolio/dashboard.py +102 -0
- mentistest_coverage-0.2.2/test_coverage_tool/portfolio/rollup.py +323 -0
- mentistest_coverage-0.2.2/test_coverage_tool/portfolio/tokens.py +130 -0
- mentistest_coverage-0.2.2/test_coverage_tool/py.typed +0 -0
- mentistest_coverage-0.2.2/test_coverage_tool/reports/__init__.py +8 -0
- mentistest_coverage-0.2.2/test_coverage_tool/reports/_format.py +160 -0
- mentistest_coverage-0.2.2/test_coverage_tool/reports/_templates/report.html.jinja +1498 -0
- mentistest_coverage-0.2.2/test_coverage_tool/reports/html_generator.py +653 -0
- mentistest_coverage-0.2.2/test_coverage_tool/reports/json_generator.py +528 -0
- mentistest_coverage-0.2.2/test_coverage_tool/reports/pdf_generator.py +442 -0
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# CHANGELOG
|
|
2
|
+
|
|
3
|
+
All notable changes to this project will be documented in this file.
|
|
4
|
+
This project adheres to [Semantic Versioning](http://semver.org/) and [Keep a Changelog](http://keepachangelog.com/).
|
|
5
|
+
|
|
6
|
+
## Unreleased
|
|
7
|
+
---
|
|
8
|
+
|
|
9
|
+
### New
|
|
10
|
+
|
|
11
|
+
### Changes
|
|
12
|
+
|
|
13
|
+
### Fixes
|
|
14
|
+
|
|
15
|
+
### Breaks
|
|
16
|
+
|
|
17
|
+
## 1.0.0
|
|
18
|
+
---
|
|
19
|
+
|
|
20
|
+
First stable release, published to PyPI as `mentistest-coverage`.
|
|
21
|
+
|
|
22
|
+
### Changes
|
|
23
|
+
|
|
24
|
+
- Renamed the distribution to `mentistest-coverage` (was `test-coverage-tool`)
|
|
25
|
+
and the console script to `mentistest` (was `test-coverage`). The importable
|
|
26
|
+
Python package remains `test_coverage_tool`, so `import test_coverage_tool`
|
|
27
|
+
is unaffected.
|
|
28
|
+
|
|
29
|
+
### Breaks
|
|
30
|
+
|
|
31
|
+
- The `test-coverage` console command is gone; use `mentistest`. Scripts that
|
|
32
|
+
shell out to `test-coverage ...` must be updated to `mentistest ...`.
|
|
33
|
+
- `pip install test-coverage-tool` no longer resolves; install
|
|
34
|
+
`mentistest-coverage` instead.
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Harry Trott
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,310 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: mentistest-coverage
|
|
3
|
+
Version: 0.2.2
|
|
4
|
+
Summary: Analyze test strategy across SaaS products with coverage metrics
|
|
5
|
+
Author-email: Harry Trott <harrydtrott@gmail.com>
|
|
6
|
+
License: MIT License
|
|
7
|
+
|
|
8
|
+
Copyright (c) 2026 Harry Trott
|
|
9
|
+
|
|
10
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
11
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
12
|
+
in the Software without restriction, including without limitation the rights
|
|
13
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
14
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
15
|
+
furnished to do so, subject to the following conditions:
|
|
16
|
+
|
|
17
|
+
The above copyright notice and this permission notice shall be included in all
|
|
18
|
+
copies or substantial portions of the Software.
|
|
19
|
+
|
|
20
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
21
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
22
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
23
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
24
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
25
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
26
|
+
SOFTWARE.
|
|
27
|
+
|
|
28
|
+
Project-URL: Homepage, https://github.com/HarryDouglas/test_coverage_tool
|
|
29
|
+
Project-URL: Repository, https://github.com/HarryDouglas/test_coverage_tool.git
|
|
30
|
+
Project-URL: Bug Tracker, https://github.com/HarryDouglas/test_coverage_tool/issues
|
|
31
|
+
Keywords: test-coverage,coverage-metrics,saas
|
|
32
|
+
Classifier: Development Status :: 4 - Beta
|
|
33
|
+
Classifier: Intended Audience :: Developers
|
|
34
|
+
Classifier: Natural Language :: English
|
|
35
|
+
Classifier: Operating System :: OS Independent
|
|
36
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
37
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
38
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
39
|
+
Requires-Python: >=3.10
|
|
40
|
+
Description-Content-Type: text/markdown
|
|
41
|
+
License-File: LICENSE
|
|
42
|
+
Requires-Dist: pydantic<3,>=2.0
|
|
43
|
+
Requires-Dist: fastapi<1,>=0.104
|
|
44
|
+
Requires-Dist: uvicorn<1,>=0.24
|
|
45
|
+
Requires-Dist: pyyaml<7,>=6.0
|
|
46
|
+
Requires-Dist: jsonschema<5,>=4.0
|
|
47
|
+
Requires-Dist: jinja2<4,>=3.1
|
|
48
|
+
Requires-Dist: typer<1,>=0.9
|
|
49
|
+
Requires-Dist: anthropic<1,>=0.40
|
|
50
|
+
Requires-Dist: httpx<1,>=0.25
|
|
51
|
+
Requires-Dist: slowapi<1,>=0.1.9
|
|
52
|
+
Requires-Dist: tomli<3,>=2.0; python_version < "3.11"
|
|
53
|
+
Provides-Extra: test
|
|
54
|
+
Requires-Dist: pytest~=9.0; extra == "test"
|
|
55
|
+
Requires-Dist: pytest-cov>=4.1; extra == "test"
|
|
56
|
+
Requires-Dist: pytest-asyncio>=0.21; extra == "test"
|
|
57
|
+
Requires-Dist: allure-pytest~=2.15; extra == "test"
|
|
58
|
+
Requires-Dist: packaging>=21.0; extra == "test"
|
|
59
|
+
Requires-Dist: hypothesis<7,>=6.0; extra == "test"
|
|
60
|
+
Provides-Extra: coverage
|
|
61
|
+
Requires-Dist: coverage>=5.0; extra == "coverage"
|
|
62
|
+
Provides-Extra: types
|
|
63
|
+
Requires-Dist: types-PyYAML>=6.0; extra == "types"
|
|
64
|
+
Provides-Extra: code-checks
|
|
65
|
+
Requires-Dist: black==25.1.0; extra == "code-checks"
|
|
66
|
+
Requires-Dist: mypy==1.16.1; extra == "code-checks"
|
|
67
|
+
Requires-Dist: pydocstyle==6.3.0; extra == "code-checks"
|
|
68
|
+
Requires-Dist: pylint==3.3.7; extra == "code-checks"
|
|
69
|
+
Requires-Dist: ruff>=0.1.0; extra == "code-checks"
|
|
70
|
+
Provides-Extra: docs
|
|
71
|
+
Requires-Dist: sphinx~=8.1; extra == "docs"
|
|
72
|
+
Requires-Dist: sphinx-breeze-theme; extra == "docs"
|
|
73
|
+
Requires-Dist: sphinxcontrib-mermaid; extra == "docs"
|
|
74
|
+
Requires-Dist: sphinx-autodoc-typehints; extra == "docs"
|
|
75
|
+
Requires-Dist: sphinx-book-theme; extra == "docs"
|
|
76
|
+
Requires-Dist: myst-parser; extra == "docs"
|
|
77
|
+
Provides-Extra: dev
|
|
78
|
+
Requires-Dist: invoke~=2.2; extra == "dev"
|
|
79
|
+
Requires-Dist: changelog-cli; extra == "dev"
|
|
80
|
+
Requires-Dist: cruft; extra == "dev"
|
|
81
|
+
Requires-Dist: ipython; extra == "dev"
|
|
82
|
+
Requires-Dist: pre-commit~=4.0; extra == "dev"
|
|
83
|
+
Requires-Dist: setuptools; extra == "dev"
|
|
84
|
+
Requires-Dist: build; extra == "dev"
|
|
85
|
+
Provides-Extra: logging-json
|
|
86
|
+
Requires-Dist: python-json-logger<4,>=2.0; extra == "logging-json"
|
|
87
|
+
Provides-Extra: metrics
|
|
88
|
+
Requires-Dist: prometheus-client<1,>=0.20; extra == "metrics"
|
|
89
|
+
Provides-Extra: all
|
|
90
|
+
Requires-Dist: invoke~=2.2; extra == "all"
|
|
91
|
+
Requires-Dist: setuptools; extra == "all"
|
|
92
|
+
Requires-Dist: build; extra == "all"
|
|
93
|
+
Dynamic: license-file
|
|
94
|
+
|
|
95
|
+
# mentistest-coverage
|
|
96
|
+
|
|
97
|
+
Strategic test-suite analyser. Looks at a Python project's source tree, test
|
|
98
|
+
tree, JUnit XML and (optionally) an OpenAPI spec, and produces a *test-strategy*
|
|
99
|
+
report — not just a coverage percentage.
|
|
100
|
+
|
|
101
|
+
What it does that `coverage.py` doesn't:
|
|
102
|
+
|
|
103
|
+
- Classifies every test as **unit / integration / contract / e2e** using markers,
|
|
104
|
+
imports, fixtures and directory layout.
|
|
105
|
+
- Weights coverage gaps by **cyclomatic complexity and call-graph depth** so the
|
|
106
|
+
20 functions that matter rank above the 200 that don't.
|
|
107
|
+
- Correlates tests against the **OpenAPI spec** to flag uncovered endpoints.
|
|
108
|
+
- Tracks results over time in a local SQLite **history store** with full git/CI
|
|
109
|
+
provenance, and renders a **trend dashboard** per project.
|
|
110
|
+
- Detects **regressions** between snapshots and can fail CI when a previously-
|
|
111
|
+
covered symbol loses coverage.
|
|
112
|
+
- Ships an opt-in **AI second opinion** (Claude Haiku → Sonnet escalation, gated
|
|
113
|
+
to skip healthy projects — typical cost <£0.01/run).
|
|
114
|
+
- Exports HTML (interactive), JSON (machine-readable) and PDF (board-ready) reports.
|
|
115
|
+
- Ships a first-class **GitHub Action** that posts a coverage-delta PR comment
|
|
116
|
+
and uploads artifacts.
|
|
117
|
+
|
|
118
|
+
Local-first. The analyser runs against filesystem paths; source code is never
|
|
119
|
+
uploaded anywhere. The optional AI call sends a structured summary (file paths,
|
|
120
|
+
coverage numbers, test names) — never the source itself.
|
|
121
|
+
|
|
122
|
+
## Install
|
|
123
|
+
|
|
124
|
+
```bash
|
|
125
|
+
pip install mentistest-coverage
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
…or from a clone:
|
|
129
|
+
|
|
130
|
+
```bash
|
|
131
|
+
pip install -e .
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
Verify:
|
|
135
|
+
|
|
136
|
+
```bash
|
|
137
|
+
mentistest --help
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
Every PyPI release is built via OIDC trusted publishing and signed
|
|
141
|
+
with Sigstore — see [RELEASING.md](RELEASING.md) for the full chain
|
|
142
|
+
and verification recipe.
|
|
143
|
+
|
|
144
|
+
## Quick start
|
|
145
|
+
|
|
146
|
+
A complete analysis with all the features wired in:
|
|
147
|
+
|
|
148
|
+
```bash
|
|
149
|
+
mentistest analyze my-service \
|
|
150
|
+
--source ./src --tests ./tests \
|
|
151
|
+
--junit ./reports/junit.xml \
|
|
152
|
+
--api-spec ./openapi.yaml \
|
|
153
|
+
--output-json ./report.json \
|
|
154
|
+
--output-html ./report.html \
|
|
155
|
+
--output-pdf ./report.pdf \
|
|
156
|
+
--track --compare-baseline --fail-on-regression \
|
|
157
|
+
--detect-git \
|
|
158
|
+
--ai
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
What that does:
|
|
162
|
+
|
|
163
|
+
| Flag | Effect |
|
|
164
|
+
| --------------------------- | ------ |
|
|
165
|
+
| `--source / -s` | Source directory (repeatable) |
|
|
166
|
+
| `--tests / -t` | Test directory (repeatable) |
|
|
167
|
+
| `--junit` | JUnit XML report path (repeatable) |
|
|
168
|
+
| `--api-spec` | OpenAPI / Swagger spec — drives endpoint-coverage analysis |
|
|
169
|
+
| `--coverage-xml` | Real coverage.xml from coverage.py (preferred over the fallback) |
|
|
170
|
+
| `--output-json / -j` | JSON report path |
|
|
171
|
+
| `--output-html / -H` | Interactive HTML dashboard path |
|
|
172
|
+
| `--output-pdf / -P` | Board-ready PDF report path |
|
|
173
|
+
| `--track / --no-track` | Save the run as a snapshot in the history DB |
|
|
174
|
+
| `--db-path` | Override `~/.test-coverage/history.db` |
|
|
175
|
+
| `--compare-baseline` | Diff this run against the most recent snapshot |
|
|
176
|
+
| `--regression-threshold N` | Allowable coverage drop in pp (default 0) |
|
|
177
|
+
| `--fail-on-regression` | `exit 1` if a regression is detected |
|
|
178
|
+
| `--detect-git / --no-detect-git` | Capture commit, branch, CI run ID, provider |
|
|
179
|
+
| `--ai` | Run the optional AI commentary (requires `ANTHROPIC_API_KEY`) |
|
|
180
|
+
| `--test-type` | Default type label when no signal fires (`unit`/`integration`/`contract`/`e2e`) |
|
|
181
|
+
| `--verbose / -v` | Debug logging |
|
|
182
|
+
|
|
183
|
+
## Server mode
|
|
184
|
+
|
|
185
|
+
```bash
|
|
186
|
+
mentistest serve --host 0.0.0.0 --port 8000
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
Because the `/analyze` endpoint reads arbitrary filesystem paths, `serve`
|
|
190
|
+
**refuses to bind a non-loopback host** (anything other than `127.0.0.1`/
|
|
191
|
+
`::1`/`localhost`) unless `TEST_COVERAGE_API_KEY` or
|
|
192
|
+
`TEST_COVERAGE_ALLOWED_PATHS` is configured. Pass `--insecure` to override
|
|
193
|
+
on a trusted private network. The published Docker image binds `0.0.0.0` and
|
|
194
|
+
sets `TEST_COVERAGE_REQUIRE_SECURITY=1`, so the container likewise won't
|
|
195
|
+
start unprotected.
|
|
196
|
+
|
|
197
|
+
Then:
|
|
198
|
+
|
|
199
|
+
- `http://localhost:8000/docs` — interactive OpenAPI docs for the REST API
|
|
200
|
+
- `http://localhost:8000/benchmarks` — multi-project benchmark dashboard
|
|
201
|
+
- `http://localhost:8000/trends/<project>` — single-project trend dashboard
|
|
202
|
+
- `http://localhost:8000/report/html?project_name=<name>` — last-rendered report
|
|
203
|
+
- `http://localhost:8000/report/pdf?project_name=<name>` — last-rendered PDF
|
|
204
|
+
- `http://localhost:8000/history/<project>` — JSON list of snapshots
|
|
205
|
+
- `http://localhost:8000/regression/<project>` — JSON regression diff
|
|
206
|
+
|
|
207
|
+
## GitHub Action
|
|
208
|
+
|
|
209
|
+
```yaml
|
|
210
|
+
- uses: HarryDouglas/test_coverage_tool@v1
|
|
211
|
+
with:
|
|
212
|
+
source_dirs: src
|
|
213
|
+
test_dirs: tests
|
|
214
|
+
junit_xml_paths: reports/junit.xml
|
|
215
|
+
track_history: 'true'
|
|
216
|
+
fail_on_regression: 'true'
|
|
217
|
+
```
|
|
218
|
+
|
|
219
|
+
Posts a PR comment with the coverage delta, uploads JSON + HTML reports as
|
|
220
|
+
workflow artifacts, and fails the check if a symbol loses coverage. See
|
|
221
|
+
`action.yml` for all inputs.
|
|
222
|
+
|
|
223
|
+
**Action vs PyPI wheel:** the action is its own distribution channel —
|
|
224
|
+
referenced by git tag, not installed from PyPI. Under the hood it
|
|
225
|
+
installs the wheel from PyPI and runs the CLI, so the two move
|
|
226
|
+
together but version independently. The action's `@v1` tag pins a
|
|
227
|
+
major; minor / patch wheel updates roll through without a workflow
|
|
228
|
+
change.
|
|
229
|
+
|
|
230
|
+
## Security
|
|
231
|
+
|
|
232
|
+
If you're deploying the HTTP API beyond `localhost`, read
|
|
233
|
+
[SECURITY.md](SECURITY.md). Two env vars (`TEST_COVERAGE_API_KEY` and
|
|
234
|
+
`TEST_COVERAGE_ALLOWED_PATHS`) enable bearer-token auth and
|
|
235
|
+
filesystem path-traversal protection. Both are off by default so local,
|
|
236
|
+
loopback-only use is unaffected — but the tool **fails closed** the moment
|
|
237
|
+
it would be network-reachable: `serve` rejects a non-loopback `--host`
|
|
238
|
+
(unless `--insecure`), and `create_app` refuses to start when
|
|
239
|
+
`TEST_COVERAGE_REQUIRE_SECURITY=1` (set in the Docker image) is configured
|
|
240
|
+
without a protection layer.
|
|
241
|
+
|
|
242
|
+
To report a vulnerability, email **harrydtrott@gmail.com** — don't
|
|
243
|
+
open a public issue.
|
|
244
|
+
|
|
245
|
+
## Public benchmark suite
|
|
246
|
+
|
|
247
|
+
The tool ships with a 16+ repo OSS benchmark suite (flask, fastapi, pydantic,
|
|
248
|
+
httpx, celery, mlflow, attrs, rich, schemathesis, dbt-core, alembic, …) used
|
|
249
|
+
both as a smoke test and as a demo gallery for the consulting site.
|
|
250
|
+
|
|
251
|
+
```bash
|
|
252
|
+
make benchmark # clone, test, analyse, render all projects
|
|
253
|
+
make benchmark-serve # serve the rendered dashboard at localhost:8003/benchmarks
|
|
254
|
+
```
|
|
255
|
+
|
|
256
|
+
Outputs land in `reports/benchmarks/<project>/` (HTML + JSON) and are
|
|
257
|
+
git-ignored. Re-run after any classifier change to verify the public pyramid
|
|
258
|
+
still looks sensible.
|
|
259
|
+
|
|
260
|
+
## Library use
|
|
261
|
+
|
|
262
|
+
```python
|
|
263
|
+
from pathlib import Path
|
|
264
|
+
from test_coverage_tool.core.analyzer import AnalysisEngine
|
|
265
|
+
from test_coverage_tool.core.git import detect_git_metadata
|
|
266
|
+
from test_coverage_tool.reports.html_generator import HTMLReportGenerator
|
|
267
|
+
from test_coverage_tool.reports.json_generator import JSONReportGenerator
|
|
268
|
+
from test_coverage_tool.history.store import HistoryStore
|
|
269
|
+
|
|
270
|
+
engine = AnalysisEngine()
|
|
271
|
+
analysis = engine.analyze(
|
|
272
|
+
project_name = "my-service",
|
|
273
|
+
source_dirs = [Path("src")],
|
|
274
|
+
test_dirs = [Path("tests")],
|
|
275
|
+
junit_xml_paths = [Path("reports/junit.xml")],
|
|
276
|
+
api_spec_path = Path("openapi.yaml"),
|
|
277
|
+
git_metadata = detect_git_metadata(),
|
|
278
|
+
)
|
|
279
|
+
JSONReportGenerator().write(analysis, Path("report.json"))
|
|
280
|
+
HTMLReportGenerator().write(analysis, Path("report.html"))
|
|
281
|
+
|
|
282
|
+
# Persist a snapshot
|
|
283
|
+
HistoryStore(Path("history.db")).save("my-service", analysis_to_dict(analysis))
|
|
284
|
+
```
|
|
285
|
+
|
|
286
|
+
## Configuration
|
|
287
|
+
|
|
288
|
+
Environment variables:
|
|
289
|
+
|
|
290
|
+
| Variable | Purpose |
|
|
291
|
+
| -------------------------- | ------- |
|
|
292
|
+
| `ANTHROPIC_API_KEY` | Enables `--ai` (otherwise the flag is a no-op) |
|
|
293
|
+
| `TEST_COVERAGE_DB_PATH` | Override the default `~/.test-coverage/history.db` location |
|
|
294
|
+
| `MENTIS_BENCH_AI` | Set to `1` to opt-in to AI commentary on `make benchmark` |
|
|
295
|
+
|
|
296
|
+
## Development
|
|
297
|
+
|
|
298
|
+
```bash
|
|
299
|
+
make install # install dev deps via uv
|
|
300
|
+
make test # pytest --cov on unittests/
|
|
301
|
+
make lint # ruff + mypy --strict + pylint
|
|
302
|
+
make benchmark # rebuild the public benchmark dashboard
|
|
303
|
+
```
|
|
304
|
+
|
|
305
|
+
The repo uses ruff, mypy (`--strict`), pylint and pytest. The quality bar:
|
|
306
|
+
zero ruff/mypy/pylint issues, pytest -x -q must pass before merge.
|
|
307
|
+
|
|
308
|
+
## License
|
|
309
|
+
|
|
310
|
+
MIT — see `LICENSE`.
|
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
# mentistest-coverage
|
|
2
|
+
|
|
3
|
+
Strategic test-suite analyser. Looks at a Python project's source tree, test
|
|
4
|
+
tree, JUnit XML and (optionally) an OpenAPI spec, and produces a *test-strategy*
|
|
5
|
+
report — not just a coverage percentage.
|
|
6
|
+
|
|
7
|
+
What it does that `coverage.py` doesn't:
|
|
8
|
+
|
|
9
|
+
- Classifies every test as **unit / integration / contract / e2e** using markers,
|
|
10
|
+
imports, fixtures and directory layout.
|
|
11
|
+
- Weights coverage gaps by **cyclomatic complexity and call-graph depth** so the
|
|
12
|
+
20 functions that matter rank above the 200 that don't.
|
|
13
|
+
- Correlates tests against the **OpenAPI spec** to flag uncovered endpoints.
|
|
14
|
+
- Tracks results over time in a local SQLite **history store** with full git/CI
|
|
15
|
+
provenance, and renders a **trend dashboard** per project.
|
|
16
|
+
- Detects **regressions** between snapshots and can fail CI when a previously-
|
|
17
|
+
covered symbol loses coverage.
|
|
18
|
+
- Ships an opt-in **AI second opinion** (Claude Haiku → Sonnet escalation, gated
|
|
19
|
+
to skip healthy projects — typical cost <£0.01/run).
|
|
20
|
+
- Exports HTML (interactive), JSON (machine-readable) and PDF (board-ready) reports.
|
|
21
|
+
- Ships a first-class **GitHub Action** that posts a coverage-delta PR comment
|
|
22
|
+
and uploads artifacts.
|
|
23
|
+
|
|
24
|
+
Local-first. The analyser runs against filesystem paths; source code is never
|
|
25
|
+
uploaded anywhere. The optional AI call sends a structured summary (file paths,
|
|
26
|
+
coverage numbers, test names) — never the source itself.
|
|
27
|
+
|
|
28
|
+
## Install
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
pip install mentistest-coverage
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
…or from a clone:
|
|
35
|
+
|
|
36
|
+
```bash
|
|
37
|
+
pip install -e .
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
Verify:
|
|
41
|
+
|
|
42
|
+
```bash
|
|
43
|
+
mentistest --help
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
Every PyPI release is built via OIDC trusted publishing and signed
|
|
47
|
+
with Sigstore — see [RELEASING.md](RELEASING.md) for the full chain
|
|
48
|
+
and verification recipe.
|
|
49
|
+
|
|
50
|
+
## Quick start
|
|
51
|
+
|
|
52
|
+
A complete analysis with all the features wired in:
|
|
53
|
+
|
|
54
|
+
```bash
|
|
55
|
+
mentistest analyze my-service \
|
|
56
|
+
--source ./src --tests ./tests \
|
|
57
|
+
--junit ./reports/junit.xml \
|
|
58
|
+
--api-spec ./openapi.yaml \
|
|
59
|
+
--output-json ./report.json \
|
|
60
|
+
--output-html ./report.html \
|
|
61
|
+
--output-pdf ./report.pdf \
|
|
62
|
+
--track --compare-baseline --fail-on-regression \
|
|
63
|
+
--detect-git \
|
|
64
|
+
--ai
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
What that does:
|
|
68
|
+
|
|
69
|
+
| Flag | Effect |
|
|
70
|
+
| --------------------------- | ------ |
|
|
71
|
+
| `--source / -s` | Source directory (repeatable) |
|
|
72
|
+
| `--tests / -t` | Test directory (repeatable) |
|
|
73
|
+
| `--junit` | JUnit XML report path (repeatable) |
|
|
74
|
+
| `--api-spec` | OpenAPI / Swagger spec — drives endpoint-coverage analysis |
|
|
75
|
+
| `--coverage-xml` | Real coverage.xml from coverage.py (preferred over the fallback) |
|
|
76
|
+
| `--output-json / -j` | JSON report path |
|
|
77
|
+
| `--output-html / -H` | Interactive HTML dashboard path |
|
|
78
|
+
| `--output-pdf / -P` | Board-ready PDF report path |
|
|
79
|
+
| `--track / --no-track` | Save the run as a snapshot in the history DB |
|
|
80
|
+
| `--db-path` | Override `~/.test-coverage/history.db` |
|
|
81
|
+
| `--compare-baseline` | Diff this run against the most recent snapshot |
|
|
82
|
+
| `--regression-threshold N` | Allowable coverage drop in pp (default 0) |
|
|
83
|
+
| `--fail-on-regression` | `exit 1` if a regression is detected |
|
|
84
|
+
| `--detect-git / --no-detect-git` | Capture commit, branch, CI run ID, provider |
|
|
85
|
+
| `--ai` | Run the optional AI commentary (requires `ANTHROPIC_API_KEY`) |
|
|
86
|
+
| `--test-type` | Default type label when no signal fires (`unit`/`integration`/`contract`/`e2e`) |
|
|
87
|
+
| `--verbose / -v` | Debug logging |
|
|
88
|
+
|
|
89
|
+
## Server mode
|
|
90
|
+
|
|
91
|
+
```bash
|
|
92
|
+
mentistest serve --host 0.0.0.0 --port 8000
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
Because the `/analyze` endpoint reads arbitrary filesystem paths, `serve`
|
|
96
|
+
**refuses to bind a non-loopback host** (anything other than `127.0.0.1`/
|
|
97
|
+
`::1`/`localhost`) unless `TEST_COVERAGE_API_KEY` or
|
|
98
|
+
`TEST_COVERAGE_ALLOWED_PATHS` is configured. Pass `--insecure` to override
|
|
99
|
+
on a trusted private network. The published Docker image binds `0.0.0.0` and
|
|
100
|
+
sets `TEST_COVERAGE_REQUIRE_SECURITY=1`, so the container likewise won't
|
|
101
|
+
start unprotected.
|
|
102
|
+
|
|
103
|
+
Then:
|
|
104
|
+
|
|
105
|
+
- `http://localhost:8000/docs` — interactive OpenAPI docs for the REST API
|
|
106
|
+
- `http://localhost:8000/benchmarks` — multi-project benchmark dashboard
|
|
107
|
+
- `http://localhost:8000/trends/<project>` — single-project trend dashboard
|
|
108
|
+
- `http://localhost:8000/report/html?project_name=<name>` — last-rendered report
|
|
109
|
+
- `http://localhost:8000/report/pdf?project_name=<name>` — last-rendered PDF
|
|
110
|
+
- `http://localhost:8000/history/<project>` — JSON list of snapshots
|
|
111
|
+
- `http://localhost:8000/regression/<project>` — JSON regression diff
|
|
112
|
+
|
|
113
|
+
## GitHub Action
|
|
114
|
+
|
|
115
|
+
```yaml
|
|
116
|
+
- uses: HarryDouglas/test_coverage_tool@v1
|
|
117
|
+
with:
|
|
118
|
+
source_dirs: src
|
|
119
|
+
test_dirs: tests
|
|
120
|
+
junit_xml_paths: reports/junit.xml
|
|
121
|
+
track_history: 'true'
|
|
122
|
+
fail_on_regression: 'true'
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
Posts a PR comment with the coverage delta, uploads JSON + HTML reports as
|
|
126
|
+
workflow artifacts, and fails the check if a symbol loses coverage. See
|
|
127
|
+
`action.yml` for all inputs.
|
|
128
|
+
|
|
129
|
+
**Action vs PyPI wheel:** the action is its own distribution channel —
|
|
130
|
+
referenced by git tag, not installed from PyPI. Under the hood it
|
|
131
|
+
installs the wheel from PyPI and runs the CLI, so the two move
|
|
132
|
+
together but version independently. The action's `@v1` tag pins a
|
|
133
|
+
major; minor / patch wheel updates roll through without a workflow
|
|
134
|
+
change.
|
|
135
|
+
|
|
136
|
+
## Security
|
|
137
|
+
|
|
138
|
+
If you're deploying the HTTP API beyond `localhost`, read
|
|
139
|
+
[SECURITY.md](SECURITY.md). Two env vars (`TEST_COVERAGE_API_KEY` and
|
|
140
|
+
`TEST_COVERAGE_ALLOWED_PATHS`) enable bearer-token auth and
|
|
141
|
+
filesystem path-traversal protection. Both are off by default so local,
|
|
142
|
+
loopback-only use is unaffected — but the tool **fails closed** the moment
|
|
143
|
+
it would be network-reachable: `serve` rejects a non-loopback `--host`
|
|
144
|
+
(unless `--insecure`), and `create_app` refuses to start when
|
|
145
|
+
`TEST_COVERAGE_REQUIRE_SECURITY=1` (set in the Docker image) is configured
|
|
146
|
+
without a protection layer.
|
|
147
|
+
|
|
148
|
+
To report a vulnerability, email **harrydtrott@gmail.com** — don't
|
|
149
|
+
open a public issue.
|
|
150
|
+
|
|
151
|
+
## Public benchmark suite
|
|
152
|
+
|
|
153
|
+
The tool ships with a 16+ repo OSS benchmark suite (flask, fastapi, pydantic,
|
|
154
|
+
httpx, celery, mlflow, attrs, rich, schemathesis, dbt-core, alembic, …) used
|
|
155
|
+
both as a smoke test and as a demo gallery for the consulting site.
|
|
156
|
+
|
|
157
|
+
```bash
|
|
158
|
+
make benchmark # clone, test, analyse, render all projects
|
|
159
|
+
make benchmark-serve # serve the rendered dashboard at localhost:8003/benchmarks
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
Outputs land in `reports/benchmarks/<project>/` (HTML + JSON) and are
|
|
163
|
+
git-ignored. Re-run after any classifier change to verify the public pyramid
|
|
164
|
+
still looks sensible.
|
|
165
|
+
|
|
166
|
+
## Library use
|
|
167
|
+
|
|
168
|
+
```python
|
|
169
|
+
from pathlib import Path
|
|
170
|
+
from test_coverage_tool.core.analyzer import AnalysisEngine
|
|
171
|
+
from test_coverage_tool.core.git import detect_git_metadata
|
|
172
|
+
from test_coverage_tool.reports.html_generator import HTMLReportGenerator
|
|
173
|
+
from test_coverage_tool.reports.json_generator import JSONReportGenerator
|
|
174
|
+
from test_coverage_tool.history.store import HistoryStore
|
|
175
|
+
|
|
176
|
+
engine = AnalysisEngine()
|
|
177
|
+
analysis = engine.analyze(
|
|
178
|
+
project_name = "my-service",
|
|
179
|
+
source_dirs = [Path("src")],
|
|
180
|
+
test_dirs = [Path("tests")],
|
|
181
|
+
junit_xml_paths = [Path("reports/junit.xml")],
|
|
182
|
+
api_spec_path = Path("openapi.yaml"),
|
|
183
|
+
git_metadata = detect_git_metadata(),
|
|
184
|
+
)
|
|
185
|
+
JSONReportGenerator().write(analysis, Path("report.json"))
|
|
186
|
+
HTMLReportGenerator().write(analysis, Path("report.html"))
|
|
187
|
+
|
|
188
|
+
# Persist a snapshot
|
|
189
|
+
HistoryStore(Path("history.db")).save("my-service", analysis_to_dict(analysis))
|
|
190
|
+
```
|
|
191
|
+
|
|
192
|
+
## Configuration
|
|
193
|
+
|
|
194
|
+
Environment variables:
|
|
195
|
+
|
|
196
|
+
| Variable | Purpose |
|
|
197
|
+
| -------------------------- | ------- |
|
|
198
|
+
| `ANTHROPIC_API_KEY` | Enables `--ai` (otherwise the flag is a no-op) |
|
|
199
|
+
| `TEST_COVERAGE_DB_PATH` | Override the default `~/.test-coverage/history.db` location |
|
|
200
|
+
| `MENTIS_BENCH_AI` | Set to `1` to opt-in to AI commentary on `make benchmark` |
|
|
201
|
+
|
|
202
|
+
## Development
|
|
203
|
+
|
|
204
|
+
```bash
|
|
205
|
+
make install # install dev deps via uv
|
|
206
|
+
make test # pytest --cov on unittests/
|
|
207
|
+
make lint # ruff + mypy --strict + pylint
|
|
208
|
+
make benchmark # rebuild the public benchmark dashboard
|
|
209
|
+
```
|
|
210
|
+
|
|
211
|
+
The repo uses ruff, mypy (`--strict`), pylint and pytest. The quality bar:
|
|
212
|
+
zero ruff/mypy/pylint issues, pytest -x -q must pass before merge.
|
|
213
|
+
|
|
214
|
+
## License
|
|
215
|
+
|
|
216
|
+
MIT — see `LICENSE`.
|