mentistest-coverage 0.2.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. mentistest_coverage-0.2.2/CHANGELOG.md +34 -0
  2. mentistest_coverage-0.2.2/LICENSE +21 -0
  3. mentistest_coverage-0.2.2/MANIFEST.in +2 -0
  4. mentistest_coverage-0.2.2/PKG-INFO +310 -0
  5. mentistest_coverage-0.2.2/README.md +216 -0
  6. mentistest_coverage-0.2.2/mentistest_coverage.egg-info/PKG-INFO +310 -0
  7. mentistest_coverage-0.2.2/mentistest_coverage.egg-info/SOURCES.txt +93 -0
  8. mentistest_coverage-0.2.2/mentistest_coverage.egg-info/dependency_links.txt +1 -0
  9. mentistest_coverage-0.2.2/mentistest_coverage.egg-info/entry_points.txt +2 -0
  10. mentistest_coverage-0.2.2/mentistest_coverage.egg-info/requires.txt +62 -0
  11. mentistest_coverage-0.2.2/mentistest_coverage.egg-info/top_level.txt +1 -0
  12. mentistest_coverage-0.2.2/pyproject.toml +374 -0
  13. mentistest_coverage-0.2.2/setup.cfg +4 -0
  14. mentistest_coverage-0.2.2/test_coverage_tool/__init__.py +17 -0
  15. mentistest_coverage-0.2.2/test_coverage_tool/_logging.py +34 -0
  16. mentistest_coverage-0.2.2/test_coverage_tool/_version.py +4 -0
  17. mentistest_coverage-0.2.2/test_coverage_tool/ai/__init__.py +8 -0
  18. mentistest_coverage-0.2.2/test_coverage_tool/ai/_scrub.py +83 -0
  19. mentistest_coverage-0.2.2/test_coverage_tool/ai/analyzer.py +455 -0
  20. mentistest_coverage-0.2.2/test_coverage_tool/ai/budget.py +177 -0
  21. mentistest_coverage-0.2.2/test_coverage_tool/ai/config.py +93 -0
  22. mentistest_coverage-0.2.2/test_coverage_tool/api/__init__.py +7 -0
  23. mentistest_coverage-0.2.2/test_coverage_tool/api/_logging_middleware.py +218 -0
  24. mentistest_coverage-0.2.2/test_coverage_tool/api/_metrics.py +131 -0
  25. mentistest_coverage-0.2.2/test_coverage_tool/api/_security.py +284 -0
  26. mentistest_coverage-0.2.2/test_coverage_tool/api/models.py +104 -0
  27. mentistest_coverage-0.2.2/test_coverage_tool/api/routes.py +661 -0
  28. mentistest_coverage-0.2.2/test_coverage_tool/benchmarks/__init__.py +7 -0
  29. mentistest_coverage-0.2.2/test_coverage_tool/benchmarks/_templates/compare.html.jinja +219 -0
  30. mentistest_coverage-0.2.2/test_coverage_tool/benchmarks/_templates/dashboard.html.jinja +977 -0
  31. mentistest_coverage-0.2.2/test_coverage_tool/benchmarks/dashboard.py +706 -0
  32. mentistest_coverage-0.2.2/test_coverage_tool/benchmarks/store.py +520 -0
  33. mentistest_coverage-0.2.2/test_coverage_tool/cli/__init__.py +7 -0
  34. mentistest_coverage-0.2.2/test_coverage_tool/cli/main.py +706 -0
  35. mentistest_coverage-0.2.2/test_coverage_tool/core/__init__.py +14 -0
  36. mentistest_coverage-0.2.2/test_coverage_tool/core/analyzer.py +1194 -0
  37. mentistest_coverage-0.2.2/test_coverage_tool/core/call_graph.py +240 -0
  38. mentistest_coverage-0.2.2/test_coverage_tool/core/classifier.py +982 -0
  39. mentistest_coverage-0.2.2/test_coverage_tool/core/classifier_facts.py +676 -0
  40. mentistest_coverage-0.2.2/test_coverage_tool/core/classifier_validation.py +400 -0
  41. mentistest_coverage-0.2.2/test_coverage_tool/core/contract_coverage.py +101 -0
  42. mentistest_coverage-0.2.2/test_coverage_tool/core/diff_coverage.py +226 -0
  43. mentistest_coverage-0.2.2/test_coverage_tool/core/endpoint_attribution.py +303 -0
  44. mentistest_coverage-0.2.2/test_coverage_tool/core/endpoint_correlation.py +364 -0
  45. mentistest_coverage-0.2.2/test_coverage_tool/core/endpoint_extractors/__init__.py +56 -0
  46. mentistest_coverage-0.2.2/test_coverage_tool/core/endpoint_extractors/class_views.py +469 -0
  47. mentistest_coverage-0.2.2/test_coverage_tool/core/endpoint_extractors/cross_check.py +205 -0
  48. mentistest_coverage-0.2.2/test_coverage_tool/core/endpoint_extractors/inbound.py +413 -0
  49. mentistest_coverage-0.2.2/test_coverage_tool/core/endpoint_extractors/javascript.py +335 -0
  50. mentistest_coverage-0.2.2/test_coverage_tool/core/endpoint_extractors/mock_libs.py +170 -0
  51. mentistest_coverage-0.2.2/test_coverage_tool/core/endpoint_extractors/outbound.py +440 -0
  52. mentistest_coverage-0.2.2/test_coverage_tool/core/endpoint_extractors/overrides.py +228 -0
  53. mentistest_coverage-0.2.2/test_coverage_tool/core/endpoint_extractors/wrappers.py +162 -0
  54. mentistest_coverage-0.2.2/test_coverage_tool/core/fixture_resolver.py +287 -0
  55. mentistest_coverage-0.2.2/test_coverage_tool/core/git.py +196 -0
  56. mentistest_coverage-0.2.2/test_coverage_tool/core/imports.py +101 -0
  57. mentistest_coverage-0.2.2/test_coverage_tool/core/javascript_classifier.py +273 -0
  58. mentistest_coverage-0.2.2/test_coverage_tool/core/marker_config.py +251 -0
  59. mentistest_coverage-0.2.2/test_coverage_tool/core/metrics.py +332 -0
  60. mentistest_coverage-0.2.2/test_coverage_tool/core/reclassify.py +503 -0
  61. mentistest_coverage-0.2.2/test_coverage_tool/core/regression.py +184 -0
  62. mentistest_coverage-0.2.2/test_coverage_tool/core/symbol_table.py +348 -0
  63. mentistest_coverage-0.2.2/test_coverage_tool/core/test_type.py +52 -0
  64. mentistest_coverage-0.2.2/test_coverage_tool/core/timing.py +223 -0
  65. mentistest_coverage-0.2.2/test_coverage_tool/core/types.py +1038 -0
  66. mentistest_coverage-0.2.2/test_coverage_tool/history/__init__.py +5 -0
  67. mentistest_coverage-0.2.2/test_coverage_tool/history/backfill.py +369 -0
  68. mentistest_coverage-0.2.2/test_coverage_tool/history/comparison.py +297 -0
  69. mentistest_coverage-0.2.2/test_coverage_tool/history/comparison_dashboard.py +353 -0
  70. mentistest_coverage-0.2.2/test_coverage_tool/history/dashboard.py +794 -0
  71. mentistest_coverage-0.2.2/test_coverage_tool/history/store.py +588 -0
  72. mentistest_coverage-0.2.2/test_coverage_tool/parsers/__init__.py +9 -0
  73. mentistest_coverage-0.2.2/test_coverage_tool/parsers/_constants.py +13 -0
  74. mentistest_coverage-0.2.2/test_coverage_tool/parsers/code_parser.py +201 -0
  75. mentistest_coverage-0.2.2/test_coverage_tool/parsers/coverage_contexts.py +172 -0
  76. mentistest_coverage-0.2.2/test_coverage_tool/parsers/coverage_parser.py +180 -0
  77. mentistest_coverage-0.2.2/test_coverage_tool/parsers/cypress_parser.py +95 -0
  78. mentistest_coverage-0.2.2/test_coverage_tool/parsers/http_traffic_parser.py +143 -0
  79. mentistest_coverage-0.2.2/test_coverage_tool/parsers/istanbul_parser.py +172 -0
  80. mentistest_coverage-0.2.2/test_coverage_tool/parsers/mutation_parser.py +209 -0
  81. mentistest_coverage-0.2.2/test_coverage_tool/parsers/pact_parser.py +227 -0
  82. mentistest_coverage-0.2.2/test_coverage_tool/parsers/spec_parser.py +118 -0
  83. mentistest_coverage-0.2.2/test_coverage_tool/parsers/test_parser.py +335 -0
  84. mentistest_coverage-0.2.2/test_coverage_tool/portfolio/__init__.py +18 -0
  85. mentistest_coverage-0.2.2/test_coverage_tool/portfolio/_templates/portfolio.html.jinja +130 -0
  86. mentistest_coverage-0.2.2/test_coverage_tool/portfolio/dashboard.py +102 -0
  87. mentistest_coverage-0.2.2/test_coverage_tool/portfolio/rollup.py +323 -0
  88. mentistest_coverage-0.2.2/test_coverage_tool/portfolio/tokens.py +130 -0
  89. mentistest_coverage-0.2.2/test_coverage_tool/py.typed +0 -0
  90. mentistest_coverage-0.2.2/test_coverage_tool/reports/__init__.py +8 -0
  91. mentistest_coverage-0.2.2/test_coverage_tool/reports/_format.py +160 -0
  92. mentistest_coverage-0.2.2/test_coverage_tool/reports/_templates/report.html.jinja +1498 -0
  93. mentistest_coverage-0.2.2/test_coverage_tool/reports/html_generator.py +653 -0
  94. mentistest_coverage-0.2.2/test_coverage_tool/reports/json_generator.py +528 -0
  95. mentistest_coverage-0.2.2/test_coverage_tool/reports/pdf_generator.py +442 -0
@@ -0,0 +1,34 @@
1
+ # CHANGELOG
2
+
3
+ All notable changes to this project will be documented in this file.
4
+ This project adheres to [Semantic Versioning](http://semver.org/) and [Keep a Changelog](http://keepachangelog.com/).
5
+
6
+ ## Unreleased
7
+ ---
8
+
9
+ ### New
10
+
11
+ ### Changes
12
+
13
+ ### Fixes
14
+
15
+ ### Breaks
16
+
17
+ ## 1.0.0
18
+ ---
19
+
20
+ First stable release, published to PyPI as `mentistest-coverage`.
21
+
22
+ ### Changes
23
+
24
+ - Renamed the distribution to `mentistest-coverage` (was `test-coverage-tool`)
25
+ and the console script to `mentistest` (was `test-coverage`). The importable
26
+ Python package remains `test_coverage_tool`, so `import test_coverage_tool`
27
+ is unaffected.
28
+
29
+ ### Breaks
30
+
31
+ - The `test-coverage` console command is gone; use `mentistest`. Scripts that
32
+ shell out to `test-coverage ...` must be updated to `mentistest ...`.
33
+ - `pip install test-coverage-tool` no longer resolves; install
34
+ `mentistest-coverage` instead.
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Harry Trott
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,2 @@
1
+ include CHANGELOG.md
2
+ include */py.typed
@@ -0,0 +1,310 @@
1
+ Metadata-Version: 2.4
2
+ Name: mentistest-coverage
3
+ Version: 0.2.2
4
+ Summary: Analyze test strategy across SaaS products with coverage metrics
5
+ Author-email: Harry Trott <harrydtrott@gmail.com>
6
+ License: MIT License
7
+
8
+ Copyright (c) 2026 Harry Trott
9
+
10
+ Permission is hereby granted, free of charge, to any person obtaining a copy
11
+ of this software and associated documentation files (the "Software"), to deal
12
+ in the Software without restriction, including without limitation the rights
13
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
14
+ copies of the Software, and to permit persons to whom the Software is
15
+ furnished to do so, subject to the following conditions:
16
+
17
+ The above copyright notice and this permission notice shall be included in all
18
+ copies or substantial portions of the Software.
19
+
20
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
23
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
25
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
26
+ SOFTWARE.
27
+
28
+ Project-URL: Homepage, https://github.com/HarryDouglas/test_coverage_tool
29
+ Project-URL: Repository, https://github.com/HarryDouglas/test_coverage_tool.git
30
+ Project-URL: Bug Tracker, https://github.com/HarryDouglas/test_coverage_tool/issues
31
+ Keywords: test-coverage,coverage-metrics,saas
32
+ Classifier: Development Status :: 4 - Beta
33
+ Classifier: Intended Audience :: Developers
34
+ Classifier: Natural Language :: English
35
+ Classifier: Operating System :: OS Independent
36
+ Classifier: Programming Language :: Python :: 3.10
37
+ Classifier: Programming Language :: Python :: 3.11
38
+ Classifier: Programming Language :: Python :: 3.12
39
+ Requires-Python: >=3.10
40
+ Description-Content-Type: text/markdown
41
+ License-File: LICENSE
42
+ Requires-Dist: pydantic<3,>=2.0
43
+ Requires-Dist: fastapi<1,>=0.104
44
+ Requires-Dist: uvicorn<1,>=0.24
45
+ Requires-Dist: pyyaml<7,>=6.0
46
+ Requires-Dist: jsonschema<5,>=4.0
47
+ Requires-Dist: jinja2<4,>=3.1
48
+ Requires-Dist: typer<1,>=0.9
49
+ Requires-Dist: anthropic<1,>=0.40
50
+ Requires-Dist: httpx<1,>=0.25
51
+ Requires-Dist: slowapi<1,>=0.1.9
52
+ Requires-Dist: tomli<3,>=2.0; python_version < "3.11"
53
+ Provides-Extra: test
54
+ Requires-Dist: pytest~=9.0; extra == "test"
55
+ Requires-Dist: pytest-cov>=4.1; extra == "test"
56
+ Requires-Dist: pytest-asyncio>=0.21; extra == "test"
57
+ Requires-Dist: allure-pytest~=2.15; extra == "test"
58
+ Requires-Dist: packaging>=21.0; extra == "test"
59
+ Requires-Dist: hypothesis<7,>=6.0; extra == "test"
60
+ Provides-Extra: coverage
61
+ Requires-Dist: coverage>=5.0; extra == "coverage"
62
+ Provides-Extra: types
63
+ Requires-Dist: types-PyYAML>=6.0; extra == "types"
64
+ Provides-Extra: code-checks
65
+ Requires-Dist: black==25.1.0; extra == "code-checks"
66
+ Requires-Dist: mypy==1.16.1; extra == "code-checks"
67
+ Requires-Dist: pydocstyle==6.3.0; extra == "code-checks"
68
+ Requires-Dist: pylint==3.3.7; extra == "code-checks"
69
+ Requires-Dist: ruff>=0.1.0; extra == "code-checks"
70
+ Provides-Extra: docs
71
+ Requires-Dist: sphinx~=8.1; extra == "docs"
72
+ Requires-Dist: sphinx-breeze-theme; extra == "docs"
73
+ Requires-Dist: sphinxcontrib-mermaid; extra == "docs"
74
+ Requires-Dist: sphinx-autodoc-typehints; extra == "docs"
75
+ Requires-Dist: sphinx-book-theme; extra == "docs"
76
+ Requires-Dist: myst-parser; extra == "docs"
77
+ Provides-Extra: dev
78
+ Requires-Dist: invoke~=2.2; extra == "dev"
79
+ Requires-Dist: changelog-cli; extra == "dev"
80
+ Requires-Dist: cruft; extra == "dev"
81
+ Requires-Dist: ipython; extra == "dev"
82
+ Requires-Dist: pre-commit~=4.0; extra == "dev"
83
+ Requires-Dist: setuptools; extra == "dev"
84
+ Requires-Dist: build; extra == "dev"
85
+ Provides-Extra: logging-json
86
+ Requires-Dist: python-json-logger<4,>=2.0; extra == "logging-json"
87
+ Provides-Extra: metrics
88
+ Requires-Dist: prometheus-client<1,>=0.20; extra == "metrics"
89
+ Provides-Extra: all
90
+ Requires-Dist: invoke~=2.2; extra == "all"
91
+ Requires-Dist: setuptools; extra == "all"
92
+ Requires-Dist: build; extra == "all"
93
+ Dynamic: license-file
94
+
95
+ # mentistest-coverage
96
+
97
+ Strategic test-suite analyser. Looks at a Python project's source tree, test
98
+ tree, JUnit XML and (optionally) an OpenAPI spec, and produces a *test-strategy*
99
+ report — not just a coverage percentage.
100
+
101
+ What it does that `coverage.py` doesn't:
102
+
103
+ - Classifies every test as **unit / integration / contract / e2e** using markers,
104
+ imports, fixtures and directory layout.
105
+ - Weights coverage gaps by **cyclomatic complexity and call-graph depth** so the
106
+ 20 functions that matter rank above the 200 that don't.
107
+ - Correlates tests against the **OpenAPI spec** to flag uncovered endpoints.
108
+ - Tracks results over time in a local SQLite **history store** with full git/CI
109
+ provenance, and renders a **trend dashboard** per project.
110
+ - Detects **regressions** between snapshots and can fail CI when a previously-
111
+ covered symbol loses coverage.
112
+ - Ships an opt-in **AI second opinion** (Claude Haiku → Sonnet escalation, gated
113
+ to skip healthy projects — typical cost <£0.01/run).
114
+ - Exports HTML (interactive), JSON (machine-readable) and PDF (board-ready) reports.
115
+ - Ships a first-class **GitHub Action** that posts a coverage-delta PR comment
116
+ and uploads artifacts.
117
+
118
+ Local-first. The analyser runs against filesystem paths; source code is never
119
+ uploaded anywhere. The optional AI call sends a structured summary (file paths,
120
+ coverage numbers, test names) — never the source itself.
121
+
122
+ ## Install
123
+
124
+ ```bash
125
+ pip install mentistest-coverage
126
+ ```
127
+
128
+ …or from a clone:
129
+
130
+ ```bash
131
+ pip install -e .
132
+ ```
133
+
134
+ Verify:
135
+
136
+ ```bash
137
+ mentistest --help
138
+ ```
139
+
140
+ Every PyPI release is built via OIDC trusted publishing and signed
141
+ with Sigstore — see [RELEASING.md](RELEASING.md) for the full chain
142
+ and verification recipe.
143
+
144
+ ## Quick start
145
+
146
+ A complete analysis with all the features wired in:
147
+
148
+ ```bash
149
+ mentistest analyze my-service \
150
+ --source ./src --tests ./tests \
151
+ --junit ./reports/junit.xml \
152
+ --api-spec ./openapi.yaml \
153
+ --output-json ./report.json \
154
+ --output-html ./report.html \
155
+ --output-pdf ./report.pdf \
156
+ --track --compare-baseline --fail-on-regression \
157
+ --detect-git \
158
+ --ai
159
+ ```
160
+
161
+ What that does:
162
+
163
+ | Flag | Effect |
164
+ | --------------------------- | ------ |
165
+ | `--source / -s` | Source directory (repeatable) |
166
+ | `--tests / -t` | Test directory (repeatable) |
167
+ | `--junit` | JUnit XML report path (repeatable) |
168
+ | `--api-spec` | OpenAPI / Swagger spec — drives endpoint-coverage analysis |
169
+ | `--coverage-xml` | Real coverage.xml from coverage.py (preferred over the fallback) |
170
+ | `--output-json / -j` | JSON report path |
171
+ | `--output-html / -H` | Interactive HTML dashboard path |
172
+ | `--output-pdf / -P` | Board-ready PDF report path |
173
+ | `--track / --no-track` | Save the run as a snapshot in the history DB |
174
+ | `--db-path` | Override `~/.test-coverage/history.db` |
175
+ | `--compare-baseline` | Diff this run against the most recent snapshot |
176
+ | `--regression-threshold N` | Allowable coverage drop in pp (default 0) |
177
+ | `--fail-on-regression` | `exit 1` if a regression is detected |
178
+ | `--detect-git / --no-detect-git` | Capture commit, branch, CI run ID, provider |
179
+ | `--ai` | Run the optional AI commentary (requires `ANTHROPIC_API_KEY`) |
180
+ | `--test-type` | Default type label when no signal fires (`unit`/`integration`/`contract`/`e2e`) |
181
+ | `--verbose / -v` | Debug logging |
182
+
183
+ ## Server mode
184
+
185
+ ```bash
186
+ mentistest serve --host 0.0.0.0 --port 8000
187
+ ```
188
+
189
+ Because the `/analyze` endpoint reads arbitrary filesystem paths, `serve`
190
+ **refuses to bind a non-loopback host** (anything other than `127.0.0.1`/
191
+ `::1`/`localhost`) unless `TEST_COVERAGE_API_KEY` or
192
+ `TEST_COVERAGE_ALLOWED_PATHS` is configured. Pass `--insecure` to override
193
+ on a trusted private network. The published Docker image binds `0.0.0.0` and
194
+ sets `TEST_COVERAGE_REQUIRE_SECURITY=1`, so the container likewise won't
195
+ start unprotected.
196
+
197
+ Then:
198
+
199
+ - `http://localhost:8000/docs` — interactive OpenAPI docs for the REST API
200
+ - `http://localhost:8000/benchmarks` — multi-project benchmark dashboard
201
+ - `http://localhost:8000/trends/<project>` — single-project trend dashboard
202
+ - `http://localhost:8000/report/html?project_name=<name>` — last-rendered report
203
+ - `http://localhost:8000/report/pdf?project_name=<name>` — last-rendered PDF
204
+ - `http://localhost:8000/history/<project>` — JSON list of snapshots
205
+ - `http://localhost:8000/regression/<project>` — JSON regression diff
206
+
207
+ ## GitHub Action
208
+
209
+ ```yaml
210
+ - uses: HarryDouglas/test_coverage_tool@v1
211
+ with:
212
+ source_dirs: src
213
+ test_dirs: tests
214
+ junit_xml_paths: reports/junit.xml
215
+ track_history: 'true'
216
+ fail_on_regression: 'true'
217
+ ```
218
+
219
+ Posts a PR comment with the coverage delta, uploads JSON + HTML reports as
220
+ workflow artifacts, and fails the check if a symbol loses coverage. See
221
+ `action.yml` for all inputs.
222
+
223
+ **Action vs PyPI wheel:** the action is its own distribution channel —
224
+ referenced by git tag, not installed from PyPI. Under the hood it
225
+ installs the wheel from PyPI and runs the CLI, so the two move
226
+ together but version independently. The action's `@v1` tag pins a
227
+ major; minor / patch wheel updates roll through without a workflow
228
+ change.
229
+
230
+ ## Security
231
+
232
+ If you're deploying the HTTP API beyond `localhost`, read
233
+ [SECURITY.md](SECURITY.md). Two env vars (`TEST_COVERAGE_API_KEY` and
234
+ `TEST_COVERAGE_ALLOWED_PATHS`) enable bearer-token auth and
235
+ filesystem path-traversal protection. Both are off by default so local,
236
+ loopback-only use is unaffected — but the tool **fails closed** the moment
237
+ it would be network-reachable: `serve` rejects a non-loopback `--host`
238
+ (unless `--insecure`), and `create_app` refuses to start when
239
+ `TEST_COVERAGE_REQUIRE_SECURITY=1` (set in the Docker image) is configured
240
+ without a protection layer.
241
+
242
+ To report a vulnerability, email **harrydtrott@gmail.com** — don't
243
+ open a public issue.
244
+
245
+ ## Public benchmark suite
246
+
247
+ The tool ships with a 16+ repo OSS benchmark suite (flask, fastapi, pydantic,
248
+ httpx, celery, mlflow, attrs, rich, schemathesis, dbt-core, alembic, …) used
249
+ both as a smoke test and as a demo gallery for the consulting site.
250
+
251
+ ```bash
252
+ make benchmark # clone, test, analyse, render all projects
253
+ make benchmark-serve # serve the rendered dashboard at localhost:8003/benchmarks
254
+ ```
255
+
256
+ Outputs land in `reports/benchmarks/<project>/` (HTML + JSON) and are
257
+ git-ignored. Re-run after any classifier change to verify the public pyramid
258
+ still looks sensible.
259
+
260
+ ## Library use
261
+
262
+ ```python
263
+ from pathlib import Path
264
+ from test_coverage_tool.core.analyzer import AnalysisEngine
265
+ from test_coverage_tool.core.git import detect_git_metadata
266
+ from test_coverage_tool.reports.html_generator import HTMLReportGenerator
267
+ from test_coverage_tool.reports.json_generator import JSONReportGenerator
268
+ from test_coverage_tool.history.store import HistoryStore
269
+
270
+ engine = AnalysisEngine()
271
+ analysis = engine.analyze(
272
+ project_name = "my-service",
273
+ source_dirs = [Path("src")],
274
+ test_dirs = [Path("tests")],
275
+ junit_xml_paths = [Path("reports/junit.xml")],
276
+ api_spec_path = Path("openapi.yaml"),
277
+ git_metadata = detect_git_metadata(),
278
+ )
279
+ JSONReportGenerator().write(analysis, Path("report.json"))
280
+ HTMLReportGenerator().write(analysis, Path("report.html"))
281
+
282
+ # Persist a snapshot
283
+ HistoryStore(Path("history.db")).save("my-service", analysis_to_dict(analysis))
284
+ ```
285
+
286
+ ## Configuration
287
+
288
+ Environment variables:
289
+
290
+ | Variable | Purpose |
291
+ | -------------------------- | ------- |
292
+ | `ANTHROPIC_API_KEY` | Enables `--ai` (otherwise the flag is a no-op) |
293
+ | `TEST_COVERAGE_DB_PATH` | Override the default `~/.test-coverage/history.db` location |
294
+ | `MENTIS_BENCH_AI` | Set to `1` to opt-in to AI commentary on `make benchmark` |
295
+
296
+ ## Development
297
+
298
+ ```bash
299
+ make install # install dev deps via uv
300
+ make test # pytest --cov on unittests/
301
+ make lint # ruff + mypy --strict + pylint
302
+ make benchmark # rebuild the public benchmark dashboard
303
+ ```
304
+
305
+ The repo uses ruff, mypy (`--strict`), pylint and pytest. The quality bar:
306
+ zero ruff/mypy/pylint issues, pytest -x -q must pass before merge.
307
+
308
+ ## License
309
+
310
+ MIT — see `LICENSE`.
@@ -0,0 +1,216 @@
1
+ # mentistest-coverage
2
+
3
+ Strategic test-suite analyser. Looks at a Python project's source tree, test
4
+ tree, JUnit XML and (optionally) an OpenAPI spec, and produces a *test-strategy*
5
+ report — not just a coverage percentage.
6
+
7
+ What it does that `coverage.py` doesn't:
8
+
9
+ - Classifies every test as **unit / integration / contract / e2e** using markers,
10
+ imports, fixtures and directory layout.
11
+ - Weights coverage gaps by **cyclomatic complexity and call-graph depth** so the
12
+ 20 functions that matter rank above the 200 that don't.
13
+ - Correlates tests against the **OpenAPI spec** to flag uncovered endpoints.
14
+ - Tracks results over time in a local SQLite **history store** with full git/CI
15
+ provenance, and renders a **trend dashboard** per project.
16
+ - Detects **regressions** between snapshots and can fail CI when a previously-
17
+ covered symbol loses coverage.
18
+ - Ships an opt-in **AI second opinion** (Claude Haiku → Sonnet escalation, gated
19
+ to skip healthy projects — typical cost <£0.01/run).
20
+ - Exports HTML (interactive), JSON (machine-readable) and PDF (board-ready) reports.
21
+ - Ships a first-class **GitHub Action** that posts a coverage-delta PR comment
22
+ and uploads artifacts.
23
+
24
+ Local-first. The analyser runs against filesystem paths; source code is never
25
+ uploaded anywhere. The optional AI call sends a structured summary (file paths,
26
+ coverage numbers, test names) — never the source itself.
27
+
28
+ ## Install
29
+
30
+ ```bash
31
+ pip install mentistest-coverage
32
+ ```
33
+
34
+ …or from a clone:
35
+
36
+ ```bash
37
+ pip install -e .
38
+ ```
39
+
40
+ Verify:
41
+
42
+ ```bash
43
+ mentistest --help
44
+ ```
45
+
46
+ Every PyPI release is built via OIDC trusted publishing and signed
47
+ with Sigstore — see [RELEASING.md](RELEASING.md) for the full chain
48
+ and verification recipe.
49
+
50
+ ## Quick start
51
+
52
+ A complete analysis with all the features wired in:
53
+
54
+ ```bash
55
+ mentistest analyze my-service \
56
+ --source ./src --tests ./tests \
57
+ --junit ./reports/junit.xml \
58
+ --api-spec ./openapi.yaml \
59
+ --output-json ./report.json \
60
+ --output-html ./report.html \
61
+ --output-pdf ./report.pdf \
62
+ --track --compare-baseline --fail-on-regression \
63
+ --detect-git \
64
+ --ai
65
+ ```
66
+
67
+ What that does:
68
+
69
+ | Flag | Effect |
70
+ | --------------------------- | ------ |
71
+ | `--source / -s` | Source directory (repeatable) |
72
+ | `--tests / -t` | Test directory (repeatable) |
73
+ | `--junit` | JUnit XML report path (repeatable) |
74
+ | `--api-spec` | OpenAPI / Swagger spec — drives endpoint-coverage analysis |
75
+ | `--coverage-xml` | Real coverage.xml from coverage.py (preferred over the fallback) |
76
+ | `--output-json / -j` | JSON report path |
77
+ | `--output-html / -H` | Interactive HTML dashboard path |
78
+ | `--output-pdf / -P` | Board-ready PDF report path |
79
+ | `--track / --no-track` | Save the run as a snapshot in the history DB |
80
+ | `--db-path` | Override `~/.test-coverage/history.db` |
81
+ | `--compare-baseline` | Diff this run against the most recent snapshot |
82
+ | `--regression-threshold N` | Allowable coverage drop in pp (default 0) |
83
+ | `--fail-on-regression` | `exit 1` if a regression is detected |
84
+ | `--detect-git / --no-detect-git` | Capture commit, branch, CI run ID, provider |
85
+ | `--ai` | Run the optional AI commentary (requires `ANTHROPIC_API_KEY`) |
86
+ | `--test-type` | Default type label when no signal fires (`unit`/`integration`/`contract`/`e2e`) |
87
+ | `--verbose / -v` | Debug logging |
88
+
89
+ ## Server mode
90
+
91
+ ```bash
92
+ mentistest serve --host 0.0.0.0 --port 8000
93
+ ```
94
+
95
+ Because the `/analyze` endpoint reads arbitrary filesystem paths, `serve`
96
+ **refuses to bind a non-loopback host** (anything other than `127.0.0.1`/
97
+ `::1`/`localhost`) unless `TEST_COVERAGE_API_KEY` or
98
+ `TEST_COVERAGE_ALLOWED_PATHS` is configured. Pass `--insecure` to override
99
+ on a trusted private network. The published Docker image binds `0.0.0.0` and
100
+ sets `TEST_COVERAGE_REQUIRE_SECURITY=1`, so the container likewise won't
101
+ start unprotected.
102
+
103
+ Then:
104
+
105
+ - `http://localhost:8000/docs` — interactive OpenAPI docs for the REST API
106
+ - `http://localhost:8000/benchmarks` — multi-project benchmark dashboard
107
+ - `http://localhost:8000/trends/<project>` — single-project trend dashboard
108
+ - `http://localhost:8000/report/html?project_name=<name>` — last-rendered report
109
+ - `http://localhost:8000/report/pdf?project_name=<name>` — last-rendered PDF
110
+ - `http://localhost:8000/history/<project>` — JSON list of snapshots
111
+ - `http://localhost:8000/regression/<project>` — JSON regression diff
112
+
113
+ ## GitHub Action
114
+
115
+ ```yaml
116
+ - uses: HarryDouglas/test_coverage_tool@v1
117
+ with:
118
+ source_dirs: src
119
+ test_dirs: tests
120
+ junit_xml_paths: reports/junit.xml
121
+ track_history: 'true'
122
+ fail_on_regression: 'true'
123
+ ```
124
+
125
+ Posts a PR comment with the coverage delta, uploads JSON + HTML reports as
126
+ workflow artifacts, and fails the check if a symbol loses coverage. See
127
+ `action.yml` for all inputs.
128
+
129
+ **Action vs PyPI wheel:** the action is its own distribution channel —
130
+ referenced by git tag, not installed from PyPI. Under the hood it
131
+ installs the wheel from PyPI and runs the CLI, so the two move
132
+ together but version independently. The action's `@v1` tag pins a
133
+ major; minor / patch wheel updates roll through without a workflow
134
+ change.
135
+
136
+ ## Security
137
+
138
+ If you're deploying the HTTP API beyond `localhost`, read
139
+ [SECURITY.md](SECURITY.md). Two env vars (`TEST_COVERAGE_API_KEY` and
140
+ `TEST_COVERAGE_ALLOWED_PATHS`) enable bearer-token auth and
141
+ filesystem path-traversal protection. Both are off by default so local,
142
+ loopback-only use is unaffected — but the tool **fails closed** the moment
143
+ it would be network-reachable: `serve` rejects a non-loopback `--host`
144
+ (unless `--insecure`), and `create_app` refuses to start when
145
+ `TEST_COVERAGE_REQUIRE_SECURITY=1` (set in the Docker image) is configured
146
+ without a protection layer.
147
+
148
+ To report a vulnerability, email **harrydtrott@gmail.com** — don't
149
+ open a public issue.
150
+
151
+ ## Public benchmark suite
152
+
153
+ The tool ships with a 16+ repo OSS benchmark suite (flask, fastapi, pydantic,
154
+ httpx, celery, mlflow, attrs, rich, schemathesis, dbt-core, alembic, …) used
155
+ both as a smoke test and as a demo gallery for the consulting site.
156
+
157
+ ```bash
158
+ make benchmark # clone, test, analyse, render all projects
159
+ make benchmark-serve # serve the rendered dashboard at localhost:8003/benchmarks
160
+ ```
161
+
162
+ Outputs land in `reports/benchmarks/<project>/` (HTML + JSON) and are
163
+ git-ignored. Re-run after any classifier change to verify the public pyramid
164
+ still looks sensible.
165
+
166
+ ## Library use
167
+
168
+ ```python
169
+ from pathlib import Path
170
+ from test_coverage_tool.core.analyzer import AnalysisEngine
171
+ from test_coverage_tool.core.git import detect_git_metadata
172
+ from test_coverage_tool.reports.html_generator import HTMLReportGenerator
173
+ from test_coverage_tool.reports.json_generator import JSONReportGenerator
174
+ from test_coverage_tool.history.store import HistoryStore
175
+
176
+ engine = AnalysisEngine()
177
+ analysis = engine.analyze(
178
+ project_name = "my-service",
179
+ source_dirs = [Path("src")],
180
+ test_dirs = [Path("tests")],
181
+ junit_xml_paths = [Path("reports/junit.xml")],
182
+ api_spec_path = Path("openapi.yaml"),
183
+ git_metadata = detect_git_metadata(),
184
+ )
185
+ JSONReportGenerator().write(analysis, Path("report.json"))
186
+ HTMLReportGenerator().write(analysis, Path("report.html"))
187
+
188
+ # Persist a snapshot
189
+ HistoryStore(Path("history.db")).save("my-service", analysis_to_dict(analysis))
190
+ ```
191
+
192
+ ## Configuration
193
+
194
+ Environment variables:
195
+
196
+ | Variable | Purpose |
197
+ | -------------------------- | ------- |
198
+ | `ANTHROPIC_API_KEY` | Enables `--ai` (otherwise the flag is a no-op) |
199
+ | `TEST_COVERAGE_DB_PATH` | Override the default `~/.test-coverage/history.db` location |
200
+ | `MENTIS_BENCH_AI` | Set to `1` to opt-in to AI commentary on `make benchmark` |
201
+
202
+ ## Development
203
+
204
+ ```bash
205
+ make install # install dev deps via uv
206
+ make test # pytest --cov on unittests/
207
+ make lint # ruff + mypy --strict + pylint
208
+ make benchmark # rebuild the public benchmark dashboard
209
+ ```
210
+
211
+ The repo uses ruff, mypy (`--strict`), pylint and pytest. The quality bar:
212
+ zero ruff/mypy/pylint issues, pytest -x -q must pass before merge.
213
+
214
+ ## License
215
+
216
+ MIT — see `LICENSE`.